Intermittent occurrence object deserialisation failure
ansell opened this issue · comments
Records in the last few hours at least have started developing the following issue, or similar. There may be an issue with the latest biocache-store release that may have caused this:
==> /var/log/tomcat9/biocache-service.log <==
2020-09-07 14:05:00,181 [http-nio-127.0.0.1-8080-exec-3964] ERROR POSO (POSO.scala:72) - Problem de-serialising value: interactions : in50 - Unrecognized token 'in50': was expecting ('true', 'false' or 'null')
at [Source: (String)"in50"; line: 1, column: 9]
com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'in50': was expecting ('true', 'false' or 'null')
at [Source: (String)"in50"; line: 1, column: 9]
at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1804)
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:703)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2853)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:1899)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:757)
at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4141)
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4000)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3042)
at au.org.ala.biocache.util.Json$.toStringArray(Json.scala:74)
at au.org.ala.biocache.poso.POSO$class.setProperty(POSO.scala:67)
at au.org.ala.biocache.model.Occurrence.setProperty(Occurrence.scala:12)
at au.org.ala.biocache.poso.CompositePOSO$class.setNestedProperty(CompositePOSO.scala:125)
at au.org.ala.biocache.model.FullRecord.setNestedProperty(FullRecord.scala:19)
at au.org.ala.biocache.poso.CompositePOSO$class.setProperty(CompositePOSO.scala:99)
at au.org.ala.biocache.model.FullRecord.setProperty(FullRecord.scala:19)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:218)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:165)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.MapLike$DefaultKeySet.foreach(MapLike.scala:174)
at au.org.ala.biocache.load.FullRecordMapper$.createFullRecord(FullRecordMapper.scala:165)
at au.org.ala.biocache.dao.OccurrenceDAOImpl.getAllVersionsByRowKey(OccurrenceDAOImpl.scala:81)
at au.org.ala.biocache.Store$.getAllVersionsByUuid(Store.scala:78)
at au.org.ala.biocache.Store.getAllVersionsByUuid(Store.scala)
at au.org.ala.biocache.util.OccurrenceUtils.getAllVersionsByUuid(OccurrenceUtils.java:37)
at au.org.ala.biocache.web.OccurrenceController.getOccurrenceInformation(OccurrenceController.java:1499)
at au.org.ala.biocache.web.OccurrenceController.showOccurrence(OccurrenceController.java:1395)
at sun.reflect.GeneratedMethodAccessor651.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:854)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:765)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at au.org.ala.biocache.web.SimpleCORSFilter.doFilter(SimpleCORSFilter.java:33)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at com.opensymphony.sitemesh.webapp.SiteMeshFilter.obtainContent(SiteMeshFilter.java:129)
at com.opensymphony.sitemesh.webapp.SiteMeshFilter.doFilter(SiteMeshFilter.java:77)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:367)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1639)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)
==> /var/log/nginx/access.log <==
[REDACTED] - - [07/Sep/2020:14:05:00 +1000] "GET /ws/occurrence/4be7c8c3-db58-4727-a993-97cbc3754a13 HTTP/1.1" 500 74 "-" "Java/1.8.0_252" "[REDACTED]" request_time=0.006 upstream_response_time=0.008 upstream_connect_time=0.000 upstream_header_time=0.008 upstream_cache_status=-
Other test cases include 45660082-f711-45b0-8335-9484ac643ce6 and 91b57ea6-9bd6-4934-bfda-2e5dc6f8c930
The error messages all seem to mention an in value, which only seems to appear in the institutionUid_p field on these records. The relevant values are:
cqlsh:occ> select "institutionUid_p" from occ.occ where rowkey IN ('879323c7-cffb-40c3-9e33-7f621d5a3c72', '45660082-f711-45b0-8335-9484ac643ce6', '91b57ea6-9bd6-4934-bfda-2e5dc6f8c930');
institutionUid_p
------------------
in22
in15
in4
(3 rows)
The issue doesn't appear to be present on biocache-ws-test with the following working:
https://biocache-ws-test.ala.org.au/ws/occurrences/4be7c8c3-db58-4727-a993-97cbc3754a13
And the following failing:
https://biocache-ws.ala.org.au/ws/occurrences/4be7c8c3-db58-4727-a993-97cbc3754a13
Production biocache-service is using biocache-store-2.4.4:
prod-1:~# ls -lah /var/lib/tomcat9/webapps-biocache-ws.ala.org.au/ws/WEB-INF/lib/ | grep biocache
-rw-r--r-- 1 tomcat tomcat 19K Nov 27 2019 biocache-jms-1.0.jar
-rw-r--r-- 1 tomcat tomcat 3.6M Nov 27 2019 biocache-store-2.4.4.jar
while test is using biocache-store-2.5.1:
test-2:~# ls -lah /var/lib/tomcat9/webapps-biocache-ws-test.ala.org.au/ws/WEB-INF/lib/ | grep biocache
-rw-r--r-- 1 tomcat tomcat 19K Nov 27 2019 biocache-jms-1.0.jar
-rw-r--r-- 1 tomcat tomcat 3.7M Aug 3 03:58 biocache-store-2.5.1.jar
There is a fairly large difference in the number of Jackson libraries between production and test, although the base versions appear to be the same:
prod-1:~# ls -lah /var/lib/tomcat9/webapps-biocache-ws.ala.org.au/ws/WEB-INF/lib/ | grep -i jackson
-rw-r--r-- 1 tomcat tomcat 66K Nov 27 2019 jackson-annotations-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 318K Nov 27 2019 jackson-core-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 227K Jul 23 02:45 jackson-core-asl-1.9.13.jar
-rw-r--r-- 1 tomcat tomcat 1.3M Nov 27 2019 jackson-databind-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 74K Nov 27 2019 jackson-dataformat-smile-2.5.4.jar
-rw-r--r-- 1 tomcat tomcat 652K Nov 27 2019 jackson-mapper-asl-1.8.5.jar
-rw-r--r-- 1 tomcat tomcat 42K Nov 27 2019 jackson-module-paranamer-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 439K Nov 27 2019 jackson-module-scala_2.10-2.9.8.jar
test-2:~# ls -lah /var/lib/tomcat9/webapps-biocache-ws-test.ala.org.au/ws/WEB-INF/lib/ | grep -i jackson
-rw-r--r-- 1 tomcat tomcat 66K Nov 27 2019 jackson-annotations-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 318K Nov 27 2019 jackson-core-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 227K Jun 25 03:58 jackson-core-asl-1.9.13.jar
-rw-r--r-- 1 tomcat tomcat 1.3M Nov 27 2019 jackson-databind-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 74K Nov 27 2019 jackson-dataformat-smile-2.5.4.jar
-rw-r--r-- 1 tomcat tomcat 26K Aug 3 03:58 jackson-datatype-guava-1.9.2.jar
-rw-r--r-- 1 tomcat tomcat 18K Aug 3 03:58 jackson-jaxrs-1.9.13.jar
-rw-r--r-- 1 tomcat tomcat 652K Nov 27 2019 jackson-mapper-asl-1.8.5.jar
-rw-r--r-- 1 tomcat tomcat 42K Nov 27 2019 jackson-module-paranamer-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 439K Nov 27 2019 jackson-module-scala_2.10-2.9.8.jar
-rw-r--r-- 1 tomcat tomcat 27K Aug 3 03:58 jackson-xc-1.9.2.jar
This is an intermittent issue, as it came back on 1 server without touching the server, and another server after rebooting. It is failing for both occurrence record view and occurrence record compare (but not occurrence search):
==> /var/log/tomcat9/biocache-service.log <==
2020-09-07 16:37:34,193 [http-nio-127.0.0.1-8080-exec-6931] ERROR POSO (POSO.scala:72) - Problem de-serialising value: interactions : in15 - Unrecognized token 'in15': was expecting ('true', 'false' or 'null')
at [Source: (String)"in15"; line: 1, column: 9]
com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'in15': was expecting ('true', 'false' or 'null')
at [Source: (String)"in15"; line: 1, column: 9]
at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1804)
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:703)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2853)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:1899)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:757)
at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4141)
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4000)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3042)
at au.org.ala.biocache.util.Json$.toStringArray(Json.scala:74)
at au.org.ala.biocache.poso.POSO$class.setProperty(POSO.scala:67)
at au.org.ala.biocache.model.Occurrence.setProperty(Occurrence.scala:12)
at au.org.ala.biocache.poso.CompositePOSO$class.setNestedProperty(CompositePOSO.scala:125)
at au.org.ala.biocache.model.FullRecord.setNestedProperty(FullRecord.scala:19)
at au.org.ala.biocache.poso.CompositePOSO$class.setProperty(CompositePOSO.scala:99)
at au.org.ala.biocache.model.FullRecord.setProperty(FullRecord.scala:19)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:218)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:165)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.MapLike$DefaultKeySet.foreach(MapLike.scala:174)
at au.org.ala.biocache.load.FullRecordMapper$.createFullRecord(FullRecordMapper.scala:165)
at au.org.ala.biocache.dao.OccurrenceDAOImpl.getAllVersionsByRowKey(OccurrenceDAOImpl.scala:81)
at au.org.ala.biocache.Store$.getAllVersionsByUuid(Store.scala:78)
at au.org.ala.biocache.Store.getAllVersionsByUuid(Store.scala)
at au.org.ala.biocache.util.OccurrenceUtils.getAllVersionsByUuid(OccurrenceUtils.java:37)
at au.org.ala.biocache.web.OccurrenceController.getOccurrenceInformation(OccurrenceController.java:1499)
at au.org.ala.biocache.web.OccurrenceController.showOccurrence(OccurrenceController.java:1395)
at sun.reflect.GeneratedMethodAccessor684.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:854)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:765)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at au.org.ala.biocache.web.SimpleCORSFilter.doFilter(SimpleCORSFilter.java:33)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at com.opensymphony.sitemesh.webapp.SiteMeshFilter.obtainContent(SiteMeshFilter.java:129)
at com.opensymphony.sitemesh.webapp.SiteMeshFilter.doFilter(SiteMeshFilter.java:77)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:367)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1639)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)
==> /var/log/nginx/access.log <==
[REDACTED] - - [07/Sep/2020:16:37:34 +1000] "GET /ws/occurrences/879323c7-cffb-40c3-9e33-7f621d5a3c72 HTTP/1.1" 500 74 "-" "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake)" "-" request_time=0.004 upstream_response_time=0.004 upstream_connect_time=0.000 upstream_header_time=0.004 upstream_cache_status=-
==> /var/log/tomcat9/biocache-service.log <==
2020-09-07 16:37:39,477 [http-nio-127.0.0.1-8080-exec-6946] INFO au.org.ala.biocache.dao.SearchDAOImpl (SearchDAOImpl.java:729) - spatial search query: lft:[287415 TO 287415]
==> /var/log/nginx/access.log <==
[REDACTED] - - [07/Sep/2020:16:37:39 +1000] "GET /ws/occurrences/search?q=lsid:urn:lsid:biodiversity.org.au:afd.taxon:dfcb918e-5b37-4b80-9856-0d3a53f53228 HTTP/1.1" 200 2777 "-" "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake)" "-" request_time=0.015 upstream_response_time=0.012 upstream_connect_time=0.000 upstream_header_time=0.012 upstream_cache_status=-
==> /var/log/tomcat9/biocache-service.log <==
2020-09-07 16:37:50,570 [http-nio-127.0.0.1-8080-exec-6945] ERROR POSO (POSO.scala:72) - Problem de-serialising value: interactions : in15 - Unrecognized token 'in15': was expecting ('true', 'false' or 'null')
at [Source: (String)"in15"; line: 1, column: 9]
com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'in15': was expecting ('true', 'false' or 'null')
at [Source: (String)"in15"; line: 1, column: 9]
at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1804)
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:703)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2853)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:1899)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:757)
at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4141)
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4000)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3042)
at au.org.ala.biocache.util.Json$.toStringArray(Json.scala:74)
at au.org.ala.biocache.poso.POSO$class.setProperty(POSO.scala:67)
at au.org.ala.biocache.model.Occurrence.setProperty(Occurrence.scala:12)
at au.org.ala.biocache.poso.CompositePOSO$class.setNestedProperty(CompositePOSO.scala:125)
at au.org.ala.biocache.model.FullRecord.setNestedProperty(FullRecord.scala:19)
at au.org.ala.biocache.poso.CompositePOSO$class.setProperty(CompositePOSO.scala:99)
at au.org.ala.biocache.model.FullRecord.setProperty(FullRecord.scala:19)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:218)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:165)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.MapLike$DefaultKeySet.foreach(MapLike.scala:174)
at au.org.ala.biocache.load.FullRecordMapper$.createFullRecord(FullRecordMapper.scala:165)
at au.org.ala.biocache.dao.OccurrenceDAOImpl.getAllVersionsByRowKey(OccurrenceDAOImpl.scala:81)
at au.org.ala.biocache.Store$.getComparisonByUuid(Store.scala:84)
at au.org.ala.biocache.Store.getComparisonByUuid(Store.scala)
at au.org.ala.biocache.util.OccurrenceUtils.getComparisonByUuid(OccurrenceUtils.java:52)
at au.org.ala.biocache.web.OccurrenceController.showOccurrence(OccurrenceController.java:1258)
at sun.reflect.GeneratedMethodAccessor721.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:854)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:765)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at au.org.ala.biocache.web.SimpleCORSFilter.doFilter(SimpleCORSFilter.java:33)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at com.opensymphony.sitemesh.webapp.SiteMeshFilter.obtainContent(SiteMeshFilter.java:129)
at com.opensymphony.sitemesh.webapp.SiteMeshFilter.doFilter(SiteMeshFilter.java:77)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:367)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1639)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)
==> /var/log/nginx/access.log <==
[REDACTED] - - [07/Sep/2020:16:37:50 +1000] "GET /ws/occurrence/compare/879323c7-cffb-40c3-9e33-7f621d5a3c72 HTTP/1.1" 500 74 "-" "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/98 Safari/537.4 (StatusCake)" "-" request_time=0.005 upstream_response_time=0.000 upstream_connect_time=0.000 upstream_header_time=0.000 upstream_cache_status=-
Two other correlations. The following was sent to /var/log/syslog rather than a tomcat-specific log file around the time the deserialisation failure started, and it showed up on each of the three search/mapping servers:
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: java.lang.NumberFormatException: For input string: "present"
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at java.lang.Integer.parseInt(Integer.java:580)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at java.lang.Integer.parseInt(Integer.java:615)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.immutable.StringLike$class.toInt(StringLike.scala:229)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.immutable.StringOps.toInt(StringOps.scala:31)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.util.Json$$anonfun$toIntArray$1.apply(Json.scala:112)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.util.Json$$anonfun$toIntArray$1.apply(Json.scala:112)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:108)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.util.Json$.toIntArray(Json.scala:112)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:186)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:165)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.Iterator$class.foreach(Iterator.scala:727)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at scala.collection.MapLike$DefaultKeySet.foreach(MapLike.scala:174)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.load.FullRecordMapper$.createFullRecord(FullRecordMapper.scala:165)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.dao.OccurrenceDAOImpl.getAllVersionsByRowKey(OccurrenceDAOImpl.scala:81)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.Store$.getComparisonByUuid(Store.scala:84)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.Store.getComparisonByUuid(Store.scala)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.util.OccurrenceUtils.getComparisonByUuid(OccurrenceUtils.java:52)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.web.OccurrenceController.showOccurrence(OccurrenceController.java:1258)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at sun.reflect.GeneratedMethodAccessor721.invoke(Unknown Source)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at java.lang.reflect.Method.invoke(Method.java:498)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:97)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:854)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:765)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at au.org.ala.biocache.web.SimpleCORSFilter.doFilter(SimpleCORSFilter.java:33)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at com.opensymphony.sitemesh.webapp.SiteMeshFilter.obtainContent(SiteMeshFilter.java:129)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at com.opensymphony.sitemesh.webapp.SiteMeshFilter.doFilter(SiteMeshFilter.java:77)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:367)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
Sep 7 13:12:07 aws-biocache-service-20-04-prod-2 tomcat9[630]: #011at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
The other correlation is the following lines from the cassandra system.log file which appears within a minute of the time the issue started to occur on each of the 4 cassandra nodes:
WARN [ScheduledTasks:1] 2020-09-07 13:10:01,299 QueryProcessor.java:106 - 95 prepared statements discarded in the last minute because cache limit reached (500 MB)
INFO [RMI TCP Connection(115719)-172.30.1.71] 2020-09-07 13:11:24,318 Keyspace.java:718 - adding secondary index table occ_uuid.occ_uuid_value_idx to operation
WARN [ScheduledTasks:1] 2020-09-07 13:12:01,299 QueryProcessor.java:106 - 9 prepared statements discarded in the last minute because cache limit reached (500 MB)
The start of the Complete Reprocess job correlates directly with the timestamps above, and it has errors matching the errors that were seen in the biocache-service log files:
aws-bstore-1b 2020-09-07 13:11:55,210 ERROR: [Cassandra3PersistenceManager] - Exception throw during paging: For input string: "present"
java.lang.NumberFormatException: For input string: "present"
at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
at java.lang.Integer.parseInt(Integer.java:580)
at java.lang.Integer.parseInt(Integer.java:615)
at scala.collection.immutable.StringLike$class.toInt(StringLike.scala:229)
at scala.collection.immutable.StringOps.toInt(StringOps.scala:31)
at au.org.ala.biocache.util.Json$$anonfun$toIntArray$1.apply(Json.scala:112)
at au.org.ala.biocache.util.Json$$anonfun$toIntArray$1.apply(Json.scala:112)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:108)
at au.org.ala.biocache.util.Json$.toIntArray(Json.scala:112)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:186)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:165)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.MapLike$DefaultKeySet.foreach(MapLike.scala:174)
at au.org.ala.biocache.load.FullRecordMapper$.createFullRecord(FullRecordMapper.scala:165)
at au.org.ala.biocache.dao.OccurrenceDAOImpl$$anonfun$pageOverRawProcessedLocal$2.apply(OccurrenceDAOImpl.scala:488)
at au.org.ala.biocache.dao.OccurrenceDAOImpl$$anonfun$pageOverRawProcessedLocal$2.apply(OccurrenceDAOImpl.scala:486)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4$$anonfun$call$2.apply(Cassandra3PersistenceManager.scala:952)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4$$anonfun$call$2.apply(Cassandra3PersistenceManager.scala:886)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4.call(Cassandra3PersistenceManager.scala:886)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4.call(Cassandra3PersistenceManager.scala:793)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
aws-bstore-1b 2020-09-07 13:11:55,215 ERROR: [POSO] - Problem de-serialising value: sounds : Unknown - Unrecognized token 'Unknown': was expecting ('true', 'false' or 'null')
at [Source: (String)"Unknown"; line: 1, column: 15]
com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Unknown': was expecting ('true', 'false' or 'null')
at [Source: (String)"Unknown"; line: 1, column: 15]
at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1804)
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:703)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2853)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:1899)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:757)
at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4141)
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4000)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3042)
at au.org.ala.biocache.util.Json$.toStringArray(Json.scala:74)
at au.org.ala.biocache.poso.POSO$class.setProperty(POSO.scala:67)
at au.org.ala.biocache.model.Occurrence.setProperty(Occurrence.scala:12)
at au.org.ala.biocache.poso.CompositePOSO$class.setNestedProperty(CompositePOSO.scala:125)
at au.org.ala.biocache.model.FullRecord.setNestedProperty(FullRecord.scala:19)
at au.org.ala.biocache.poso.CompositePOSO$class.setProperty(CompositePOSO.scala:99)
at au.org.ala.biocache.model.FullRecord.setProperty(FullRecord.scala:19)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:218)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:165)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.MapLike$DefaultKeySet.foreach(MapLike.scala:174)
at au.org.ala.biocache.load.FullRecordMapper$.createFullRecord(FullRecordMapper.scala:165)
at au.org.ala.biocache.dao.OccurrenceDAOImpl$$anonfun$pageOverRawProcessedLocal$2.apply(OccurrenceDAOImpl.scala:488)
at au.org.ala.biocache.dao.OccurrenceDAOImpl$$anonfun$pageOverRawProcessedLocal$2.apply(OccurrenceDAOImpl.scala:486)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4$$anonfun$call$2.apply(Cassandra3PersistenceManager.scala:952)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4$$anonfun$call$2.apply(Cassandra3PersistenceManager.scala:886)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4.call(Cassandra3PersistenceManager.scala:886)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4.call(Cassandra3PersistenceManager.scala:793)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
aws-bstore-1b 2020-09-07 13:11:55,244 ERROR: [POSO] - Problem de-serialising value: interactions : in34 - Unrecognized token 'in34': was expecting ('true', 'false' or 'null')
at [Source: (String)"in34"; line: 1, column: 9]
com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'in34': was expecting ('true', 'false' or 'null')
at [Source: (String)"in34"; line: 1, column: 9]
at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1804)
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:703)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2853)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:1899)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:757)
at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4141)
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4000)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3042)
at au.org.ala.biocache.util.Json$.toStringArray(Json.scala:74)
at au.org.ala.biocache.poso.POSO$class.setProperty(POSO.scala:67)
at au.org.ala.biocache.model.Occurrence.setProperty(Occurrence.scala:12)
at au.org.ala.biocache.poso.CompositePOSO$class.setNestedProperty(CompositePOSO.scala:125)
at au.org.ala.biocache.model.FullRecord.setNestedProperty(FullRecord.scala:19)
at au.org.ala.biocache.poso.CompositePOSO$class.setProperty(CompositePOSO.scala:99)
at au.org.ala.biocache.model.FullRecord.setProperty(FullRecord.scala:19)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:218)
at au.org.ala.biocache.load.FullRecordMapper$$anonfun$createFullRecord$3.apply(FullRecordMapper.scala:165)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.MapLike$DefaultKeySet.foreach(MapLike.scala:174)
at au.org.ala.biocache.load.FullRecordMapper$.createFullRecord(FullRecordMapper.scala:165)
at au.org.ala.biocache.dao.OccurrenceDAOImpl$$anonfun$pageOverRawProcessedLocal$2.apply(OccurrenceDAOImpl.scala:488)
at au.org.ala.biocache.dao.OccurrenceDAOImpl$$anonfun$pageOverRawProcessedLocal$2.apply(OccurrenceDAOImpl.scala:486)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4$$anonfun$call$2.apply(Cassandra3PersistenceManager.scala:952)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4$$anonfun$call$2.apply(Cassandra3PersistenceManager.scala:886)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4.call(Cassandra3PersistenceManager.scala:886)
at au.org.ala.biocache.persistence.Cassandra3PersistenceManager$$anonfun$pageOverLocalNotAsync$1$$anon$4.call(Cassandra3PersistenceManager.scala:793)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
The Complete Reprocess also possibly tried to add a field to the occ.occ table in cassandra around the time that queries started breaking:
aws-bstore-1b 2020-09-07 13:11:48,207 ERROR: [Cassandra3PersistenceManager] - adding missing column 'georeferenceRemarks_p' to 'occ' for error: Undefined column name "georeferenceRemarks_p"
The field addition appears to be novel, as I can't find other references to similar events in the log file:
91618:INFO [Native-Transport-Requests-1] 2020-09-07 13:11:48,212 MigrationManager.java:446 - Update table 'occ/occ' From org.apache.cassandra.config.CFMetaData@5549636b[cfId=6add86e0-5ce1-11e8-aaa4-45c10a4073a9,ksName=occ,cfName=occ,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apache.cassandra.db.compaction.LeveledCompactionStrategy, options={sstable_size_in_mb=500}}, compression=org.apache.cassandra.schema.CompressionParams@e733dc5e, extensions={}, cdc=false},comparator=comparator(),partitionColumns=[[] | [IdentificationQualifierProcessor_qa _class _qa abcdIdentificationQualifier abcdIdentificationQualifierInsertionPoint abcdIdentificationQualifier_p abcdTypeStatus acceptedNameUsage acceptedNameUsageID accessRights alaUserId associatedMedia associatedOccurrences associatedOccurrences_p associatedReferences associatedReferences_p associatedSequences associatedTaxa attr_qa austConservation_p australianHerbariumRegion basisOfRecord basisOfRecord_p bbox bbox_p behavior bibliographicCitation biome biome_p bor_qa catalogNumber citation cl_p class classID classID_p class_p class_qa classs classs_p collectionCode collectionCode_p collectionID collectionName collectionName_p collectionUid collectionUid_p collectorFieldNumber continent continent_p coordinatePrecision coordinatePrecision_p coordinateUncertaintyInMeters coordinateUncertaintyInMeters_p country countryCode countryCode_p countryConservation countryConservation_p country_p county cultivarName cultivated dataGeneralizations dataGeneralizations_p dataHubName dataHubUid dataHubUid_p dataProviderName dataProviderName_p dataProviderUid dataProviderUid_p dataResourceName dataResourceName_p dataResourceUid datasetID datasetName dateDeleted dateIdentified dateIdentified_p datePrecision datePrecision_p day day_p decimalLatitude decimalLatitude_p decimalLatitudelatitude decimalLongitude decimalLongitude_p defaultValuesUsed default_qa deleted disposition distanceOutsideExpertRange distanceOutsideExpertRange_p duplicates duplicatesOriginalInstitutionID duplicatesOriginalUnitID duplicates_qa duplicationStatus duplicationStatus_p duplicationType duplicationType_p dynamicProperties easting easting_p el_p endDayOfYear endYear establishmentMeans establishmentMeans_p eventAttributes eventDate eventDateEnd eventDateEnd_p eventDate_p eventID eventRemarks eventTime event_qa family familyID familyID_p family_p fieldNotes fieldNumber firstLoaded footprintSRS footprintSpatialFit footprintWKT generalisationToApplyInMetres generalisedLocality genus genusID genusID_p genus_p geodeticDatum geodeticDatum_p georeferenceProtocol georeferenceProtocol_p georeferenceRemarks georeferenceSources georeferenceSources_p georeferenceVerificationStatus georeferenceVerificationStatus_p georeferencedBy georeferencedBy_p georeferencedDate georeferencedDate_p geospatialIssue geospatiallyKosher globalConservation gridReference habitat habitat_p higherClassification higherGeography higherGeographyID ibra ibraSubregion ibra_p identificationID identificationQualifier identificationQualifier_p identificationReferences identificationReferences_p identificationRemarks identificationVerificationStatus identificationVerificationStatus_p identification_qa identifiedBy identifiedBy_p identifier identifierBy identifierRole image_qa images images_p imcra_p individualCount individualID informationWithheld informationWithheld_p infraspecificEpithet infraspecificEpithet_p infraspecificMarker institutionCode institutionCode_p institutionID institutionID_p institutionName institutionName_p institutionUid institutionUid_p interactions interactions_p island islandGroup kingdom kingdomID kingdomID_p kingdom_p language lastModifiedTime lastModifiedTime_p lastUserAssertionDate left left_p lga lga_p license license_p lifeStage lifeStage_p loanDate loanDestination loanForBotanist loanIdentifier loanReturnDate loanSequenceNumber loc_qa locality locality_p locationAccordingTo locationAttributes locationDetermined locationID locationRemarks maximumDepthInMeters maximumDepthInMeters_p maximumDistanceAboveSurfaceInMeters maximumElevationInMeters maximumElevationInMeters_p measurementAccuracy measurementDeterminedBy measurementDeterminedDate measurementID measurementMethod measurementRemarks measurementType measurementUnit measurementValue minimumDepthInMeters minimumDepthInMeters_p minimumDistanceAboveSurfaceInMeters minimumElevationInMeters minimumElevationInMeters_p miscProperties modified modified_p month month_p municipality mytest nameAccordingTo nameAccordingToID nameAccordingTo_p nameMatchMetric nameMatchMetric_p nameParseType nameParseType_p namePublishedIn namePublishedInID namePublishedInYear naturalOccurrence nearNamedPlaceRelationTo nomenclaturalCode nomenclaturalCode_p nomenclaturalStatus northing northing_p occurrenceAttributes occurrenceDetails occurrenceID occurrenceRemarks occurrenceStatus occurrenceStatus_p offline_qa order orderID orderID_p order_p organismQuantity organismQuantityType originalDecimalLatitude originalDecimalLongitude originalNameUsage originalNameUsageID originalSensitiveValues otherCatalogNumbers outlierForLayers outlierForLayers_p ownerInstitutionCode parentNameUsage parentNameUsageID phenology photoPageUrl photographer phylum phylumID phylumID_p phylum_p pointRadiusSpatialFit portalId preferredFlag preparations previousIdentifications provenance provenance_p qualityAssertion queryAssertions_p recordNumber recordedBy recordedBy_p relatedResourceID relationshipAccordingTo relationshipEstablishedDate relationshipOfResource relationshipRemarks reprocessing_qa reproductiveCondition resourceID resourceRelationshipID right right_p rights rightsholder samplingEffort samplingProtocol samplingProtocol_p scientificName scientificNameAddendum scientificNameAuthorship scientificNameAuthorship_p scientificNameID scientificNameWithoutAuthor scientificName_p secondaryCollectors sensitive_qa sex sounds sounds_p source species speciesGroups speciesGroups_p speciesHabitats speciesHabitats_p speciesID speciesID_p species_p specificEpithet specificEpithet_p startDayOfYear startYear state stateConservation stateConservation_p stateProvince stateProvince_p subfamily subgenus subgenusID subspecies subspeciesID subspeciesID_p subspecies_p superfamily taxonConceptID taxonConceptID_p taxonID taxonRank taxonRankID taxonRankID_p taxonRank_p taxonRemarks taxonomicIssue taxonomicIssue_p taxonomicStatus taxonomicallyKosher type typeStatus typeStatusQualifier typeStatus_p type_qa typifiedName userAssertionStatus userId userId_p userQualityAssertion userVerified validDistribution verbatimCoordinateSystem verbatimCoordinates verbatimDateIdentified verbatimDepth verbatimDepth_p verbatimElevation verbatimElevation_p verbatimEventDate verbatimLatitude verbatimLocality verbatimLongitude verbatimSRS verbatimTaxonRank verbatimTaxonRank_p verificationDate verificationNotes verifier vernacularName vernacularName_p videos videos_p waterBody year year_p zone]],partitionKeyColumns=[rowkey],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.UTF8Type,columnMetadata=[footprintSpatialFit, ownerInstitutionCode, license_p, verificationDate, biome_p, dataGeneralizations_p, identifier, endDayOfYear, interactions, geodeticDatum, identifierBy, georeferenceSources_p, interactions_p, occurrenceDetails, sounds, loanDate, coordinatePrecision_p, familyID_p, scientificNameAddendum, rights, userAssertionStatus, globalConservation, habitat_p, informationWithheld, habitat, collectionName_p, locationID, collectionUid_p, duplicatesOriginalUnitID, nameParseType_p, identifierRole, firstLoaded, minimumElevationInMeters_p, right_p, country, eventRemarks, dataResourceUid, relationshipOfResource, municipality, georeferenceProtocol_p, typifiedName, userId_p, higherGeography, nomenclaturalStatus, ibra_p, duplicatesOriginalInstitutionID, associatedOccurrences, decimalLongitude, infraspecificEpithet, reprocessing_qa, verbatimTaxonRank, family, scientificNameID, eventID, bor_qa, collectionName, associatedSequences, citation, provenance_p, loanReturnDate, australianHerbariumRegion, institutionName, associatedTaxa, taxonRemarks, speciesGroups, type, bbox, scientificNameAuthorship, decimalLongitude_p, portalId, occurrenceStatus_p, speciesID_p, IdentificationQualifierProcessor_qa, loanIdentifier, images_p, _class, genus, taxonomicIssue, locality_p, loanForBotanist, datePrecision, dataResourceName_p, speciesID, eventTime, loc_qa, dataHubUid, verbatimElevation, nomenclaturalCode_p, abcdIdentificationQualifier_p, verbatimSRS, distanceOutsideExpertRange_p, measurementValue, countryConservation_p, bibliographicCitation, georeferencedDate_p, typeStatusQualifier, dateIdentified, subspecies, default_qa, dynamicProperties, georeferencedBy_p, catalogNumber, subspeciesID, easting, locationAccordingTo, institutionCode, behavior, establishmentMeans, associatedReferences_p, dataHubUid_p, duplicates, geospatiallyKosher, classID_p, reproductiveCondition, institutionCode_p, right, locationDetermined, nameAccordingToID, geospatialIssue, originalNameUsageID, establishmentMeans_p, left_p, infraspecificEpithet_p, acceptedNameUsage, alaUserId, nameAccordingTo, minimumDepthInMeters_p, infraspecificMarker, recordNumber, stateProvince_p, informationWithheld_p, acceptedNameUsageID, resourceRelationshipID, lga_p, footprintWKT, secondaryCollectors, scientificName_p, dataProviderUid_p, higherGeographyID, occurrenceID, lga, sensitive_qa, georeferenceProtocol, queryAssertions_p, abcdIdentificationQualifierInsertionPoint, identificationQualifier_p, basisOfRecord, verbatimLatitude, verbatimTaxonRank_p, disposition, miscProperties, kingdom_p, individualCount, verbatimLongitude, month, genus_p, duplicates_qa, stateConservation, duplicationStatus_p, superfamily, kingdomID, otherCatalogNumbers, decimalLatitudelatitude, license, associatedOccurrences_p, modified, photographer, associatedReferences, country_p, abcdIdentificationQualifier, eventDateEnd_p, event_qa, year_p, higherClassification, dateDeleted, locality, endYear, phylum, verbatimDateIdentified, taxonRankID, validDistribution, rowkey, samplingEffort, type_qa, maximumElevationInMeters_p, identifiedBy, loanDestination, subfamily, genusID_p, dataHubName, maximumDepthInMeters_p, cultivarName, northing_p, originalDecimalLatitude, basisOfRecord_p, previousIdentifications, islandGroup, coordinatePrecision, fieldNotes, waterBody, originalNameUsage, namePublishedInYear, collectionUid, taxonConceptID, species, continent_p, georeferenceVerificationStatus, maximumDepthInMeters, provenance, samplingProtocol_p, taxonRankID_p, orderID, cultivated, identification_qa, austConservation_p, generalisationToApplyInMetres, image_qa, nameParseType, locationAttributes, kingdom, phenology, occurrenceAttributes, verbatimCoordinates, month_p, maximumElevationInMeters, measurementType, deleted, measurementDeterminedBy, samplingProtocol, speciesHabitats, classs, bbox_p, stateConservation_p, qualityAssertion, recordedBy, offline_qa, taxonomicStatus, relatedResourceID, eventDate_p, fieldNumber, left, lastModifiedTime_p, relationshipRemarks, generalisedLocality, countryCode, identificationRemarks, images, measurementRemarks, collectionCode, ibraSubregion, vernacularName_p, modified_p, phylum_p, verbatimCoordinateSystem, speciesHabitats_p, coordinateUncertaintyInMeters_p, eventAttributes, identificationReferences, continent, outlierForLayers_p, _qa, lifeStage, measurementID, typeStatus, classID, datePrecision_p, measurementDeterminedDate, institutionUid_p, phylumID, userVerified, verbatimLocality, coordinateUncertaintyInMeters, day_p, identificationID, nomenclaturalCode, subgenusID, scientificNameWithoutAuthor, classs_p, duplicationType_p, footprintSRS, minimumElevationInMeters, source, taxonRank_p, dataProviderName, day, nameMatchMetric_p, originalDecimalLongitude, rightsholder, userQualityAssertion, dataProviderName_p, northing, individualID, nearNamedPlaceRelationTo, occurrenceRemarks, abcdTypeStatus, videos, locationRemarks, class_p, geodeticDatum_p, relationshipAccordingTo, minimumDistanceAboveSurfaceInMeters, relationshipEstablishedDate, videos_p, attr_qa, decimalLatitude_p, taxonConceptID_p, stateProvince, year, organismQuantity, dateIdentified_p, collectionCode_p, naturalOccurrence, speciesGroups_p, vernacularName, lifeStage_p, georeferencedBy, order, identificationQualifier, duplicationStatus, identificationReferences_p, identificationVerificationStatus, pointRadiusSpatialFit, taxonomicIssue_p, institutionID_p, taxonID, dataGeneralizations, measurementMethod, verbatimDepth, mytest, verificationNotes, taxonRank, loanSequenceNumber, verbatimElevation_p, dataResourceName, datasetName, measurementUnit, countryConservation, namePublishedIn, georeferenceVerificationStatus_p, recordedBy_p, subspeciesID_p, startYear, userId, county, institutionID, organismQuantityType, sounds_p, island, state, maximumDistanceAboveSurfaceInMeters, georeferenceSources, language, nameAccordingTo_p, verbatimEventDate, genusID, resourceID, outlierForLayers, family_p, institutionName_p, startDayOfYear, institutionUid, dataProviderUid, cl_p, datasetID, distanceOutsideExpertRange, el_p, specificEpithet_p, occurrenceStatus, verifier, identificationVerificationStatus_p, minimumDepthInMeters, preferredFlag, parentNameUsageID, collectorFieldNumber, orderID_p, ibra, sex, species_p, originalSensitiveValues, easting_p, defaultValuesUsed, eventDateEnd, preparations, associatedMedia, georeferencedDate, nameMatchMetric, countryCode_p, georeferenceRemarks, identifiedBy_p, photoPageUrl, eventDate, collectionID, lastUserAssertionDate, parentNameUsage, specificEpithet, verbatimDepth_p, taxonomicallyKosher, class_qa, measurementAccuracy, subspecies_p, biome, namePublishedInID, subgenus, scientificNameAuthorship_p, lastModifiedTime, gridReference, scientificName, decimalLatitude, duplicationType, class, zone, accessRights, kingdomID_p, typeStatus_p, order_p, phylumID_p, familyID, imcra_p],droppedColumns={java.nio.HeapByteBuffer[pos=0 lim=9 cap=9]=DroppedColumn{name=orderID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470869914000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=duplicationStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470829000000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=georeferencedDate_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470844392000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=dataGeneralizations_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1530512099140000}, java.nio.HeapByteBuffer[pos=0 lim=26 cap=26]=DroppedColumn{name=identificationReferences_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470847094000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=georeferencedBy_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470841986000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=outlierForLayers_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470871058000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=samplingProtocol_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470876323000}, java.nio.HeapByteBuffer[pos=0 lim=23 cap=23]=DroppedColumn{name=decimalLatitudelatitude, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1528471506485000}, java.nio.HeapByteBuffer[pos=0 lim=23 cap=23]=DroppedColumn{name=associatedOccurrences_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470799444000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=taxonomicIssue_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470890078000}, java.nio.HeapByteBuffer[pos=0 lim=13 cap=13]=DroppedColumn{name=taxonRankID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470887981000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=datePrecision_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470823932000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=institutionID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470853311000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=dataGeneralizations, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1530512092520000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=institutionUid_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470856045000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=austConservation_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470802035000}, java.nio.HeapByteBuffer[pos=0 lim=13 cap=13]=DroppedColumn{name=countryCode_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470812195000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=georeferenceSources_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470839077000}, java.nio.HeapByteBuffer[pos=0 lim=14 cap=14]=DroppedColumn{name=eventDateEnd_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470832281000}, java.nio.HeapByteBuffer[pos=0 lim=22 cap=22]=DroppedColumn{name=associatedReferences_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470800440000}, java.nio.HeapByteBuffer[pos=0 lim=14 cap=14]=DroppedColumn{name=identifiedBy_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470850036000}, java.nio.HeapByteBuffer[pos=0 lim=12 cap=12]=DroppedColumn{name=dataHubUid_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470817110000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=coordinatePrecision_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470809047000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=verbatimElevation_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470894924000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=duplicationType_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470829336000}, java.nio.HeapByteBuffer[pos=0 lim=9 cap=9]=DroppedColumn{name=classID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470804394000}, java.nio.HeapByteBuffer[pos=0 lim=10 cap=10]=DroppedColumn{name=familyID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470834381000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=taxonRank_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470888926000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=verbatimDepth_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470893952000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=institutionCode_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470852975000}, java.nio.HeapByteBuffer[pos=0 lim=32 cap=32]=DroppedColumn{name=georeferenceVerificationStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470841041000}, java.nio.HeapByteBuffer[pos=0 lim=12 cap=12]=DroppedColumn{name=recordedBy_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470874942000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=nameMatchMetric_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470866082000}, java.nio.HeapByteBuffer[pos=0 lim=14 cap=14]=DroppedColumn{name=subspeciesID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470886061000}, java.nio.HeapByteBuffer[pos=0 lim=20 cap=20]=DroppedColumn{name=establishmentMeans_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470831096000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=dataProviderName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470818260000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=decimalLongitude_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470826060000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=occurrenceStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470868360000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=speciesID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470880946000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=collectionName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470806917000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=geodeticDatum_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470836954000}, java.nio.HeapByteBuffer[pos=0 lim=22 cap=22]=DroppedColumn{name=minimumDepthInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470863006000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=nameAccordingTo_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470865108000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=collectionUid_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470808103000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=stateConservation_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470882308000}, java.nio.HeapByteBuffer[pos=0 lim=26 cap=26]=DroppedColumn{name=maximumElevationInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470862025000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=stateProvince_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470883280000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=nameParseType_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470867032000}, java.nio.HeapByteBuffer[pos=0 lim=9 cap=9]=DroppedColumn{name=genusID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470835161000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=institutionName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470855069000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=informationWithheld_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470852003000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=dataResourceName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470821014000}, java.nio.HeapByteBuffer[pos=0 lim=22 cap=22]=DroppedColumn{name=maximumDepthInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470861077000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=eventDate_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470833433000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=basisOfRecord_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470803195000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=decimalLatitude_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470825112000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=taxonConceptID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470886399000}, java.nio.HeapByteBuffer[pos=0 lim=26 cap=26]=DroppedColumn{name=minimumElevationInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470863954000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=scientificName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470877468000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=dataProviderUid_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470820037000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=lastModifiedTime_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470858136000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=nomenclaturalCode_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470867359000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=queryAssertions_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470873995000}, java.nio.HeapByteBuffer[pos=0 lim=25 cap=25]=DroppedColumn{name=identificationQualifier_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470845948000}, java.nio.HeapByteBuffer[pos=0 lim=29 cap=29]=DroppedColumn{name=abcdIdentificationQualifier_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470798004000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=dateIdentified_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470822372000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=verbatimTaxonRank_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470896080000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=kingdomID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470856988000}, java.nio.HeapByteBuffer[pos=0 lim=28 cap=28]=DroppedColumn{name=distanceOutsideExpertRange_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470828023000}, java.nio.HeapByteBuffer[pos=0 lim=12 cap=12]=DroppedColumn{name=typeStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470891045000}, java.nio.HeapByteBuffer[pos=0 lim=10 cap=10]=DroppedColumn{name=phylumID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470872240000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=vernacularName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470898035000}, java.nio.HeapByteBuffer[pos=0 lim=34 cap=34]=DroppedColumn{name=identificationVerificationStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470848076000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=collectionCode_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470805959000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=speciesGroups_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470878417000}, java.nio.HeapByteBuffer[pos=0 lim=22 cap=22]=DroppedColumn{name=georeferenceProtocol_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470838104000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=speciesHabitats_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470880007000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=lifeStage_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470859927000}, java.nio.HeapByteBuffer[pos=0 lim=31 cap=31]=DroppedColumn{name=coordinateUncertaintyInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470811010000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=countryConservation_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470813342000}, java.nio.HeapByteBuffer[pos=0 lim=8 cap=8]=DroppedColumn{name=userId_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470891987000}},triggers=[],indexes=[]] To org.apache.cassandra.config.CFMetaData@486abb83[cfId=6add86e0-5ce1-11e8-aaa4-45c10a4073a9,ksName=occ,cfName=occ,flags=[COMPOUND],params=TableParams{comment=, read_repair_chance=0.0, dclocal_read_repair_chance=0.1, bloom_filter_fp_chance=0.01, crc_check_chance=1.0, gc_grace_seconds=864000, default_time_to_live=0, memtable_flush_period_in_ms=0, min_index_interval=128, max_index_interval=2048, speculative_retry=99PERCENTILE, caching={'keys' : 'ALL', 'rows_per_partition' : 'NONE'}, compaction=CompactionParams{class=org.apache.cassandra.db.compaction.LeveledCompactionStrategy, options={sstable_size_in_mb=500}}, compression=org.apache.cassandra.schema.CompressionParams@e733dc5e, extensions={}, cdc=false},comparator=comparator(),partitionColumns=[[] | [IdentificationQualifierProcessor_qa _class _qa abcdIdentificationQualifier abcdIdentificationQualifierInsertionPoint abcdIdentificationQualifier_p abcdTypeStatus acceptedNameUsage acceptedNameUsageID accessRights alaUserId associatedMedia associatedOccurrences associatedOccurrences_p associatedReferences associatedReferences_p associatedSequences associatedTaxa attr_qa austConservation_p australianHerbariumRegion basisOfRecord basisOfRecord_p bbox bbox_p behavior bibliographicCitation biome biome_p bor_qa catalogNumber citation cl_p class classID classID_p class_p class_qa classs classs_p collectionCode collectionCode_p collectionID collectionName collectionName_p collectionUid collectionUid_p collectorFieldNumber continent continent_p coordinatePrecision coordinatePrecision_p coordinateUncertaintyInMeters coordinateUncertaintyInMeters_p country countryCode countryCode_p countryConservation countryConservation_p country_p county cultivarName cultivated dataGeneralizations dataGeneralizations_p dataHubName dataHubUid dataHubUid_p dataProviderName dataProviderName_p dataProviderUid dataProviderUid_p dataResourceName dataResourceName_p dataResourceUid datasetID datasetName dateDeleted dateIdentified dateIdentified_p datePrecision datePrecision_p day day_p decimalLatitude decimalLatitude_p decimalLatitudelatitude decimalLongitude decimalLongitude_p defaultValuesUsed default_qa deleted disposition distanceOutsideExpertRange distanceOutsideExpertRange_p duplicates duplicatesOriginalInstitutionID duplicatesOriginalUnitID duplicates_qa duplicationStatus duplicationStatus_p duplicationType duplicationType_p dynamicProperties easting easting_p el_p endDayOfYear endYear establishmentMeans establishmentMeans_p eventAttributes eventDate eventDateEnd eventDateEnd_p eventDate_p eventID eventRemarks eventTime event_qa family familyID familyID_p family_p fieldNotes fieldNumber firstLoaded footprintSRS footprintSpatialFit footprintWKT generalisationToApplyInMetres generalisedLocality genus genusID genusID_p genus_p geodeticDatum geodeticDatum_p georeferenceProtocol georeferenceProtocol_p georeferenceRemarks georeferenceRemarks_p georeferenceSources georeferenceSources_p georeferenceVerificationStatus georeferenceVerificationStatus_p georeferencedBy georeferencedBy_p georeferencedDate georeferencedDate_p geospatialIssue geospatiallyKosher globalConservation gridReference habitat habitat_p higherClassification higherGeography higherGeographyID ibra ibraSubregion ibra_p identificationID identificationQualifier identificationQualifier_p identificationReferences identificationReferences_p identificationRemarks identificationVerificationStatus identificationVerificationStatus_p identification_qa identifiedBy identifiedBy_p identifier identifierBy identifierRole image_qa images images_p imcra_p individualCount individualID informationWithheld informationWithheld_p infraspecificEpithet infraspecificEpithet_p infraspecificMarker institutionCode institutionCode_p institutionID institutionID_p institutionName institutionName_p institutionUid institutionUid_p interactions interactions_p island islandGroup kingdom kingdomID kingdomID_p kingdom_p language lastModifiedTime lastModifiedTime_p lastUserAssertionDate left left_p lga lga_p license license_p lifeStage lifeStage_p loanDate loanDestination loanForBotanist loanIdentifier loanReturnDate loanSequenceNumber loc_qa locality locality_p locationAccordingTo locationAttributes locationDetermined locationID locationRemarks maximumDepthInMeters maximumDepthInMeters_p maximumDistanceAboveSurfaceInMeters maximumElevationInMeters maximumElevationInMeters_p measurementAccuracy measurementDeterminedBy measurementDeterminedDate measurementID measurementMethod measurementRemarks measurementType measurementUnit measurementValue minimumDepthInMeters minimumDepthInMeters_p minimumDistanceAboveSurfaceInMeters minimumElevationInMeters minimumElevationInMeters_p miscProperties modified modified_p month month_p municipality mytest nameAccordingTo nameAccordingToID nameAccordingTo_p nameMatchMetric nameMatchMetric_p nameParseType nameParseType_p namePublishedIn namePublishedInID namePublishedInYear naturalOccurrence nearNamedPlaceRelationTo nomenclaturalCode nomenclaturalCode_p nomenclaturalStatus northing northing_p occurrenceAttributes occurrenceDetails occurrenceID occurrenceRemarks occurrenceStatus occurrenceStatus_p offline_qa order orderID orderID_p order_p organismQuantity organismQuantityType originalDecimalLatitude originalDecimalLongitude originalNameUsage originalNameUsageID originalSensitiveValues otherCatalogNumbers outlierForLayers outlierForLayers_p ownerInstitutionCode parentNameUsage parentNameUsageID phenology photoPageUrl photographer phylum phylumID phylumID_p phylum_p pointRadiusSpatialFit portalId preferredFlag preparations previousIdentifications provenance provenance_p qualityAssertion queryAssertions_p recordNumber recordedBy recordedBy_p relatedResourceID relationshipAccordingTo relationshipEstablishedDate relationshipOfResource relationshipRemarks reprocessing_qa reproductiveCondition resourceID resourceRelationshipID right right_p rights rightsholder samplingEffort samplingProtocol samplingProtocol_p scientificName scientificNameAddendum scientificNameAuthorship scientificNameAuthorship_p scientificNameID scientificNameWithoutAuthor scientificName_p secondaryCollectors sensitive_qa sex sounds sounds_p source species speciesGroups speciesGroups_p speciesHabitats speciesHabitats_p speciesID speciesID_p species_p specificEpithet specificEpithet_p startDayOfYear startYear state stateConservation stateConservation_p stateProvince stateProvince_p subfamily subgenus subgenusID subspecies subspeciesID subspeciesID_p subspecies_p superfamily taxonConceptID taxonConceptID_p taxonID taxonRank taxonRankID taxonRankID_p taxonRank_p taxonRemarks taxonomicIssue taxonomicIssue_p taxonomicStatus taxonomicallyKosher type typeStatus typeStatusQualifier typeStatus_p type_qa typifiedName userAssertionStatus userId userId_p userQualityAssertion userVerified validDistribution verbatimCoordinateSystem verbatimCoordinates verbatimDateIdentified verbatimDepth verbatimDepth_p verbatimElevation verbatimElevation_p verbatimEventDate verbatimLatitude verbatimLocality verbatimLongitude verbatimSRS verbatimTaxonRank verbatimTaxonRank_p verificationDate verificationNotes verifier vernacularName vernacularName_p videos videos_p waterBody year year_p zone]],partitionKeyColumns=[rowkey],clusteringColumns=[],keyValidator=org.apache.cassandra.db.marshal.UTF8Type,columnMetadata=[footprintSpatialFit, ownerInstitutionCode, license_p, verificationDate, biome_p, dataGeneralizations_p, identifier, endDayOfYear, interactions, geodeticDatum, identifierBy, georeferenceSources_p, interactions_p, occurrenceDetails, sounds, loanDate, coordinatePrecision_p, familyID_p, scientificNameAddendum, rights, userAssertionStatus, globalConservation, habitat_p, informationWithheld, habitat, collectionName_p, locationID, collectionUid_p, duplicatesOriginalUnitID, nameParseType_p, identifierRole, firstLoaded, minimumElevationInMeters_p, right_p, country, eventRemarks, dataResourceUid, relationshipOfResource, municipality, georeferenceProtocol_p, typifiedName, userId_p, higherGeography, nomenclaturalStatus, ibra_p, duplicatesOriginalInstitutionID, associatedOccurrences, decimalLongitude, infraspecificEpithet, reprocessing_qa, verbatimTaxonRank, family, georeferenceRemarks_p, scientificNameID, eventID, bor_qa, collectionName, associatedSequences, citation, provenance_p, loanReturnDate, australianHerbariumRegion, institutionName, associatedTaxa, taxonRemarks, speciesGroups, type, bbox, scientificNameAuthorship, decimalLongitude_p, portalId, occurrenceStatus_p, speciesID_p, IdentificationQualifierProcessor_qa, loanIdentifier, images_p, _class, genus, taxonomicIssue, locality_p, loanForBotanist, datePrecision, dataResourceName_p, speciesID, eventTime, loc_qa, dataHubUid, verbatimElevation, nomenclaturalCode_p, abcdIdentificationQualifier_p, verbatimSRS, distanceOutsideExpertRange_p, measurementValue, countryConservation_p, bibliographicCitation, georeferencedDate_p, typeStatusQualifier, dateIdentified, subspecies, default_qa, dynamicProperties, georeferencedBy_p, catalogNumber, subspeciesID, easting, locationAccordingTo, institutionCode, behavior, establishmentMeans, associatedReferences_p, dataHubUid_p, duplicates, geospatiallyKosher, classID_p, reproductiveCondition, institutionCode_p, right, locationDetermined, nameAccordingToID, geospatialIssue, originalNameUsageID, establishmentMeans_p, left_p, infraspecificEpithet_p, acceptedNameUsage, alaUserId, nameAccordingTo, minimumDepthInMeters_p, infraspecificMarker, recordNumber, stateProvince_p, informationWithheld_p, acceptedNameUsageID, resourceRelationshipID, lga_p, footprintWKT, secondaryCollectors, scientificName_p, dataProviderUid_p, higherGeographyID, occurrenceID, lga, sensitive_qa, georeferenceProtocol, queryAssertions_p, abcdIdentificationQualifierInsertionPoint, identificationQualifier_p, basisOfRecord, verbatimLatitude, verbatimTaxonRank_p, disposition, miscProperties, kingdom_p, individualCount, verbatimLongitude, month, genus_p, duplicates_qa, stateConservation, duplicationStatus_p, superfamily, kingdomID, otherCatalogNumbers, decimalLatitudelatitude, license, associatedOccurrences_p, modified, photographer, associatedReferences, country_p, abcdIdentificationQualifier, eventDateEnd_p, event_qa, year_p, higherClassification, dateDeleted, locality, endYear, phylum, verbatimDateIdentified, taxonRankID, validDistribution, rowkey, samplingEffort, type_qa, maximumElevationInMeters_p, identifiedBy, loanDestination, subfamily, genusID_p, dataHubName, maximumDepthInMeters_p, cultivarName, northing_p, originalDecimalLatitude, basisOfRecord_p, previousIdentifications, islandGroup, coordinatePrecision, fieldNotes, waterBody, originalNameUsage, namePublishedInYear, collectionUid, taxonConceptID, species, continent_p, georeferenceVerificationStatus, maximumDepthInMeters, provenance, samplingProtocol_p, taxonRankID_p, orderID, cultivated, identification_qa, austConservation_p, generalisationToApplyInMetres, image_qa, nameParseType, locationAttributes, kingdom, phenology, occurrenceAttributes, verbatimCoordinates, month_p, maximumElevationInMeters, measurementType, deleted, measurementDeterminedBy, samplingProtocol, speciesHabitats, classs, bbox_p, stateConservation_p, qualityAssertion, recordedBy, offline_qa, taxonomicStatus, relatedResourceID, eventDate_p, fieldNumber, left, lastModifiedTime_p, relationshipRemarks, generalisedLocality, countryCode, identificationRemarks, images, measurementRemarks, collectionCode, ibraSubregion, vernacularName_p, modified_p, phylum_p, verbatimCoordinateSystem, speciesHabitats_p, coordinateUncertaintyInMeters_p, eventAttributes, identificationReferences, continent, outlierForLayers_p, _qa, lifeStage, measurementID, typeStatus, classID, datePrecision_p, measurementDeterminedDate, institutionUid_p, phylumID, userVerified, verbatimLocality, coordinateUncertaintyInMeters, day_p, identificationID, nomenclaturalCode, subgenusID, scientificNameWithoutAuthor, classs_p, duplicationType_p, footprintSRS, minimumElevationInMeters, source, taxonRank_p, dataProviderName, day, nameMatchMetric_p, originalDecimalLongitude, rightsholder, userQualityAssertion, dataProviderName_p, northing, individualID, nearNamedPlaceRelationTo, occurrenceRemarks, abcdTypeStatus, videos, locationRemarks, class_p, geodeticDatum_p, relationshipAccordingTo, minimumDistanceAboveSurfaceInMeters, relationshipEstablishedDate, videos_p, attr_qa, decimalLatitude_p, taxonConceptID_p, stateProvince, year, organismQuantity, dateIdentified_p, collectionCode_p, naturalOccurrence, speciesGroups_p, vernacularName, lifeStage_p, georeferencedBy, order, identificationQualifier, duplicationStatus, identificationReferences_p, identificationVerificationStatus, pointRadiusSpatialFit, taxonomicIssue_p, institutionID_p, taxonID, dataGeneralizations, measurementMethod, verbatimDepth, mytest, verificationNotes, taxonRank, loanSequenceNumber, verbatimElevation_p, dataResourceName, datasetName, measurementUnit, countryConservation, namePublishedIn, georeferenceVerificationStatus_p, recordedBy_p, subspeciesID_p, startYear, userId, county, institutionID, organismQuantityType, sounds_p, island, state, maximumDistanceAboveSurfaceInMeters, georeferenceSources, language, nameAccordingTo_p, verbatimEventDate, genusID, resourceID, outlierForLayers, family_p, institutionName_p, startDayOfYear, institutionUid, dataProviderUid, cl_p, datasetID, distanceOutsideExpertRange, el_p, specificEpithet_p, occurrenceStatus, verifier, identificationVerificationStatus_p, minimumDepthInMeters, preferredFlag, parentNameUsageID, collectorFieldNumber, orderID_p, ibra, sex, species_p, originalSensitiveValues, easting_p, defaultValuesUsed, eventDateEnd, preparations, associatedMedia, georeferencedDate, nameMatchMetric, countryCode_p, georeferenceRemarks, identifiedBy_p, photoPageUrl, eventDate, collectionID, lastUserAssertionDate, parentNameUsage, specificEpithet, verbatimDepth_p, taxonomicallyKosher, class_qa, measurementAccuracy, subspecies_p, biome, namePublishedInID, subgenus, scientificNameAuthorship_p, lastModifiedTime, gridReference, scientificName, decimalLatitude, duplicationType, class, zone, accessRights, kingdomID_p, typeStatus_p, order_p, phylumID_p, familyID, imcra_p],droppedColumns={java.nio.HeapByteBuffer[pos=0 lim=9 cap=9]=DroppedColumn{name=orderID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470869914000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=duplicationStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470829000000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=georeferencedDate_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470844392000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=dataGeneralizations_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1530512099140000}, java.nio.HeapByteBuffer[pos=0 lim=26 cap=26]=DroppedColumn{name=identificationReferences_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470847094000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=georeferencedBy_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470841986000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=outlierForLayers_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470871058000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=samplingProtocol_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470876323000}, java.nio.HeapByteBuffer[pos=0 lim=23 cap=23]=DroppedColumn{name=decimalLatitudelatitude, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1528471506485000}, java.nio.HeapByteBuffer[pos=0 lim=23 cap=23]=DroppedColumn{name=associatedOccurrences_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470799444000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=taxonomicIssue_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470890078000}, java.nio.HeapByteBuffer[pos=0 lim=13 cap=13]=DroppedColumn{name=taxonRankID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470887981000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=datePrecision_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470823932000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=institutionID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470853311000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=dataGeneralizations, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1530512092520000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=institutionUid_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470856045000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=austConservation_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470802035000}, java.nio.HeapByteBuffer[pos=0 lim=13 cap=13]=DroppedColumn{name=countryCode_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470812195000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=georeferenceSources_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470839077000}, java.nio.HeapByteBuffer[pos=0 lim=14 cap=14]=DroppedColumn{name=eventDateEnd_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470832281000}, java.nio.HeapByteBuffer[pos=0 lim=22 cap=22]=DroppedColumn{name=associatedReferences_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470800440000}, java.nio.HeapByteBuffer[pos=0 lim=14 cap=14]=DroppedColumn{name=identifiedBy_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470850036000}, java.nio.HeapByteBuffer[pos=0 lim=12 cap=12]=DroppedColumn{name=dataHubUid_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470817110000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=coordinatePrecision_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470809047000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=verbatimElevation_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470894924000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=duplicationType_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470829336000}, java.nio.HeapByteBuffer[pos=0 lim=9 cap=9]=DroppedColumn{name=classID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470804394000}, java.nio.HeapByteBuffer[pos=0 lim=10 cap=10]=DroppedColumn{name=familyID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470834381000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=taxonRank_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470888926000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=verbatimDepth_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470893952000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=institutionCode_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470852975000}, java.nio.HeapByteBuffer[pos=0 lim=32 cap=32]=DroppedColumn{name=georeferenceVerificationStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470841041000}, java.nio.HeapByteBuffer[pos=0 lim=12 cap=12]=DroppedColumn{name=recordedBy_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470874942000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=nameMatchMetric_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470866082000}, java.nio.HeapByteBuffer[pos=0 lim=14 cap=14]=DroppedColumn{name=subspeciesID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470886061000}, java.nio.HeapByteBuffer[pos=0 lim=20 cap=20]=DroppedColumn{name=establishmentMeans_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470831096000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=dataProviderName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470818260000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=decimalLongitude_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470826060000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=occurrenceStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470868360000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=speciesID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470880946000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=collectionName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470806917000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=geodeticDatum_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470836954000}, java.nio.HeapByteBuffer[pos=0 lim=22 cap=22]=DroppedColumn{name=minimumDepthInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470863006000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=nameAccordingTo_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470865108000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=collectionUid_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470808103000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=stateConservation_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470882308000}, java.nio.HeapByteBuffer[pos=0 lim=26 cap=26]=DroppedColumn{name=maximumElevationInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470862025000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=stateProvince_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470883280000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=nameParseType_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470867032000}, java.nio.HeapByteBuffer[pos=0 lim=9 cap=9]=DroppedColumn{name=genusID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470835161000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=institutionName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470855069000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=informationWithheld_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470852003000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=dataResourceName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470821014000}, java.nio.HeapByteBuffer[pos=0 lim=22 cap=22]=DroppedColumn{name=maximumDepthInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470861077000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=eventDate_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470833433000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=basisOfRecord_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470803195000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=decimalLatitude_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470825112000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=taxonConceptID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470886399000}, java.nio.HeapByteBuffer[pos=0 lim=26 cap=26]=DroppedColumn{name=minimumElevationInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470863954000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=scientificName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470877468000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=dataProviderUid_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470820037000}, java.nio.HeapByteBuffer[pos=0 lim=18 cap=18]=DroppedColumn{name=lastModifiedTime_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470858136000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=nomenclaturalCode_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470867359000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=queryAssertions_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470873995000}, java.nio.HeapByteBuffer[pos=0 lim=25 cap=25]=DroppedColumn{name=identificationQualifier_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470845948000}, java.nio.HeapByteBuffer[pos=0 lim=29 cap=29]=DroppedColumn{name=abcdIdentificationQualifier_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470798004000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=dateIdentified_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470822372000}, java.nio.HeapByteBuffer[pos=0 lim=19 cap=19]=DroppedColumn{name=verbatimTaxonRank_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470896080000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=kingdomID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470856988000}, java.nio.HeapByteBuffer[pos=0 lim=28 cap=28]=DroppedColumn{name=distanceOutsideExpertRange_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470828023000}, java.nio.HeapByteBuffer[pos=0 lim=12 cap=12]=DroppedColumn{name=typeStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470891045000}, java.nio.HeapByteBuffer[pos=0 lim=10 cap=10]=DroppedColumn{name=phylumID_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470872240000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=vernacularName_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470898035000}, java.nio.HeapByteBuffer[pos=0 lim=34 cap=34]=DroppedColumn{name=identificationVerificationStatus_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470848076000}, java.nio.HeapByteBuffer[pos=0 lim=16 cap=16]=DroppedColumn{name=collectionCode_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470805959000}, java.nio.HeapByteBuffer[pos=0 lim=15 cap=15]=DroppedColumn{name=speciesGroups_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470878417000}, java.nio.HeapByteBuffer[pos=0 lim=22 cap=22]=DroppedColumn{name=georeferenceProtocol_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470838104000}, java.nio.HeapByteBuffer[pos=0 lim=17 cap=17]=DroppedColumn{name=speciesHabitats_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470880007000}, java.nio.HeapByteBuffer[pos=0 lim=11 cap=11]=DroppedColumn{name=lifeStage_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470859927000}, java.nio.HeapByteBuffer[pos=0 lim=31 cap=31]=DroppedColumn{name=coordinateUncertaintyInMeters_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470811010000}, java.nio.HeapByteBuffer[pos=0 lim=21 cap=21]=DroppedColumn{name=countryConservation_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470813342000}, java.nio.HeapByteBuffer[pos=0 lim=8 cap=8]=DroppedColumn{name=userId_p, kind=null, type=org.apache.cassandra.db.marshal.UTF8Type, droppedTime=1529470891987000}},triggers=[],indexes=[]]
This commit is the likely cause. Possibly useful to manually add new fields out of peak query time rather than relying on the automatic addition:
Closing this as wont-fix because we are migrating off cassaandra soon so it doesn't make sense to add a new method for safely creating a new occ.occ field at this point.