NoClassDefFoundError for ObjectFlattener exception in Tranquility when using Avro plugin
skysaeed opened this issue · comments
runnig Tranquility
bin/tranquility kafka -configFile ../druid-0.12.1/conf-quickstart/tranquility/avro.json
with parameters in application.ini for Tranquility conf:
-Ddruid.extensions.directory=/Users/foo/bar/druid-0.12.1/extensions
-Ddruid.extensions.loadList=["druid-avro-extensions"]
and the relevant avro.json is :
{
"dataSources" : {
"pv-avro-kafka" : {
"spec" : {
"dataSchema" : {
"dataSource" : "pv-avro-kafka",
"parser" : {
"type" : "avro_stream",
"avroBytesDecoder" : {
"type" : "schema_inline",
"schema" : {
"namespace": "SKY",
"name": "mypageviews",
"type": "record",
"fields": [
{ "name": "time", "type": "string" },
{ "name": "url", "type": "string" },
{ "name": "user", "type": "string" },
{ "name": "latencyMs", "type": "int" }
]
}
},
"parseSpec" : {
"timestampSpec" : {
"column" : "time",
"format" : "auto"
},
"dimensionsSpec" : {
"dimensions" : ["url", "user"],
"dimensionExclusions" : [
"timestamp",
"value"
]
},
"format" : "avro"
}
},
"granularitySpec" : {
"type" : "uniform",
"segmentGranularity" : "hour",
"queryGranularity" : "none"
},
"metricsSpec" : [
{"name": "views", "type": "count"},
{"name": "latencyMs", "type": "doubleSum", "fieldName": "latencyMs"}
]
},
"ioConfig" : {
"type" : "realtime"
},
"tuningConfig" : {
"type" : "realtime",
"maxRowsInMemory" : "100000",
"intermediatePersistPeriod" : "PT10M",
"windowPeriod" : "PT10M"
}
},
"properties" : {
"task.partitions" : "1",
"task.replicants" : "1",
"topicPattern" : "pageviews"
}
}
},
"properties" : {
"zookeeper.connect" : "localhost:2181",
"druid.discovery.curator.path" : "/druid/discovery",
"druid.selectors.indexing.serviceName" : "druid/overlord",
"commit.periodMillis" : "15000",
"consumer.numThreads" : "2",
"kafka.zookeeper.connect" : "localhost:2181",
"kafka.group.id" : "tranquility-kafka"
}
}when I produce Avro events on Kafka using:
bin/kafka-avro-console-producer --broker-list localhost:9092 --topic pageviews --property value.schema='{"type":"record","name":"mypageviews","fields":[{"name":"time","type":"string"},{"name":"url","type":"string"},{"name":"user","type":"string"},{"name":"latencyMs","type":"int"}]}'
{"time": "2018-05-31T14:23:11Z", "url": "/foo/bar", "user": "alice", "latencyMs": 32}I get this error:
[KafkaConsumer-0] ERROR c.m.tranquility.kafka.KafkaConsumer - Exception:
java.lang.NoClassDefFoundError: Lio/druid/java/util/common/parsers/ObjectFlattener;
here is the more detailed Tranquility log:
...
INFO io.druid.guice.JsonConfigurator - Loaded class[class io.druid.guice.ExtensionsConfig] from props[druid.extensions.] as [ExtensionsConfig{searchCurrentClassloader=true, directory='/Users/foo/bar/druid-0.12.1/extensions', hadoopDependenciesDir='hadoop-dependencies', hadoopContainerDruidClasspath='null', loadList=[druid-avro-extensions]}]
...
ERROR c.m.tranquility.kafka.KafkaConsumer - Exception:
java.lang.NoClassDefFoundError: Lio/druid/java/util/common/parsers/ObjectFlattener;
at java.lang.Class.getDeclaredFields0(Native Method) ~[na:1.8.0_162]
at java.lang.Class.privateGetDeclaredFields(Class.java:2583) ~[na:1.8.0_162]
at java.lang.Class.getDeclaredFields(Class.java:1916) ~[na:1.8.0_162]
at com.fasterxml.jackson.databind.introspect.AnnotatedClass._findFields(AnnotatedClass.java:689) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.introspect.AnnotatedClass.resolveFields(AnnotatedClass.java:470) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.introspect.AnnotatedClass.fields(AnnotatedClass.java:282) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.introspect.POJOPropertiesCollector._addFields(POJOPropertiesCollector.java:384) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.introspect.POJOPropertiesCollector.collect(POJOPropertiesCollector.java:237) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.introspect.BasicClassIntrospector.collectProperties(BasicClassIntrospector.java:142) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.introspect.BasicClassIntrospector.forDeserialization(BasicClassIntrospector.java:81) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.introspect.BasicClassIntrospector.forDeserialization(BasicClassIntrospector.java:11) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.DeserializationConfig.introspect(DeserializationConfig.java:550) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.deser.DeserializerCache._createDeserializer(DeserializerCache.java:327) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.deser.DeserializerCache._createAndCache2(DeserializerCache.java:264) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.deser.DeserializerCache._createAndCacheValueDeserializer(DeserializerCache.java:244) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.deser.DeserializerCache.findValueDeserializer(DeserializerCache.java:142) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
at com.fasterxml.jackson.databind.DeserializationContext.findContextualValueDeserializer(DeserializationContext.java:380) ~[com.fasterxml.jackson.core.jackson-databind-2.4.6.jar:2.4.6]
tranquility_log.log
this is log as a file.
I tried to add missing java-util-0.12.1.jar( in which ObjectFlattener is defined) to ClassPath. Then NoClassDefFoundError was solved but then another error occurred:
[KafkaConsumer-0] ERROR c.m.tranquility.kafka.KafkaConsumer - Exception:
java.lang.AbstractMethodError: io.druid.data.input.AvroStreamInputRowParser.parse(Ljava/lang/Object;)Lio/druid/data/input/InputRow;
at com.metamx.tranquility.druid.DruidBeams$$anonfun$1$$anonfun$7.apply(DruidBeams.scala:182) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.druid.DruidBeams$$anonfun$1$$anonfun$7.apply(DruidBeams.scala:182) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.druid.DruidBeams$$anonfun$1$$anonfun$apply$1.apply(DruidBeams.scala:200) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.druid.DruidBeams$$anonfun$1$$anonfun$apply$1.apply(DruidBeams.scala:200) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.beam.TransformingBeam$$anonfun$sendAll$2$$anonfun$2.apply(TransformingBeam.scala:36) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.twitter.util.Try$.apply(Try.scala:13) ~[com.twitter.util-core_2.11-6.30.0.jar:6.30.0]
at com.metamx.tranquility.beam.TransformingBeam$$anonfun$sendAll$2.apply(TransformingBeam.scala:36) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.beam.TransformingBeam$$anonfun$sendAll$2.apply(TransformingBeam.scala:35) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:778) ~[org.scala-lang.scala-library-2.11.7.jar:na]
at scala.collection.Iterator$class.foreach(Iterator.scala:742) ~[org.scala-lang.scala-library-2.11.7.jar:na]
at scala.collection.AbstractIterator.foreach(Iterator.scala:1194) ~[org.scala-lang.scala-library-2.11.7.jar:na]
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) ~[org.scala-lang.scala-library-2.11.7.jar:na]
at scala.collection.AbstractIterable.foreach(Iterable.scala:54) ~[org.scala-lang.scala-library-2.11.7.jar:na]
at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:777) ~[org.scala-lang.scala-library-2.11.7.jar:na]
at com.metamx.tranquility.beam.TransformingBeam.sendAll(TransformingBeam.scala:35) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.tranquilizer.Tranquilizer.com$metamx$tranquility$tranquilizer$Tranquilizer$$sendBuffer(Tranquilizer.scala:301) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.tranquilizer.Tranquilizer$$anonfun$send$1.apply(Tranquilizer.scala:202) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.tranquilizer.Tranquilizer$$anonfun$send$1.apply(Tranquilizer.scala:202) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at scala.Option.foreach(Option.scala:257) ~[org.scala-lang.scala-library-2.11.7.jar:na]
at com.metamx.tranquility.tranquilizer.Tranquilizer.send(Tranquilizer.scala:202) ~[io.druid.tranquility-core-0.8.2.jar:0.8.2]
at com.metamx.tranquility.kafka.writer.TranquilityEventWriter.send(TranquilityEventWriter.java:76) ~[io.druid.tranquility-kafka-0.8.2.jar:0.8.2]
at com.metamx.tranquility.kafka.KafkaConsumer$2.run(KafkaConsumer.java:231) ~[io.druid.tranquility-kafka-0.8.2.jar:0.8.2]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_162]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_162]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_162]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_162]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_162]
2018-06-13 10:33:50,025 [KafkaConsumer-0] INFO c.m.tranquility.kafka.KafkaConsumer - Shutting down - attempting to flush buffers and commit final offsets