Service[druid:overlord] call failed with status: 400 Bad Request

Question

Service[druid:overlord] call failed with status: 400 Bad Request

amoussoubaruch opened this issue 7 years ago · comments

Hello,

I installed druid in cluster mode last week and now i'm trying to put kafaka data topic in one datasource.
I'm using tranquility API for spark and i write a job spark to take data in kafka and push that in druid.
So i have a cluster with 3 nodes
Node 1 :

overlord
coordinator
middleManager

Node 2 :

historical node
middleManager
brocker

Node 3 : Like node 2

When i start my spark job i have this error

java.lang.IllegalStateException: Failed to save new beam for identifier[druid:overlord/foo1107] timestamp[2017-07-19T08:00:00.000Z] at com.metamx.tranquility.beam.ClusteredBeam$$anonfun$2.applyOrElse(ClusteredBeam.scala:289) at com.metamx.tranquility.beam.ClusteredBeam$$anonfun$2.applyOrElse(ClusteredBeam.scala:286) at com.twitter.util.Future$$anonfun$rescue$1.apply(Future.scala:924) at com.twitter.util.Future$$anonfun$rescue$1.apply(Future.scala:922) at com.twitter.util.Promise$Transformer.liftedTree1$1(Promise.scala:112) at com.twitter.util.Promise$Transformer.k(Promise.scala:112) at com.twitter.util.Promise$Transformer.apply(Promise.scala:122) at com.twitter.util.Promise$Transformer.apply(Promise.scala:103) at com.twitter.util.Promise$$anon$1.run(Promise.scala:366) at com.twitter.concurrent.LocalScheduler$Activation.run(Scheduler.scala:178) at com.twitter.concurrent.LocalScheduler$Activation.submit(Scheduler.scala:136) at com.twitter.concurrent.LocalScheduler.submit(Scheduler.scala:207) at com.twitter.concurrent.Scheduler$.submit(Scheduler.scala:92) at com.twitter.util.Promise.runq(Promise.scala:350) at com.twitter.util.Promise.updateIfEmpty(Promise.scala:721) at com.twitter.util.ExecutorServiceFuturePool$$anon$2.run(FuturePool.scala:107) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: com.metamx.tranquility.druid.IndexServicePermanentException: Service[druid:overlord] call failed with status: 400 Bad Request at com.metamx.tranquility.druid.IndexService$$anonfun$call$1$$anonfun$apply$17.apply(IndexService.scala:158) at com.metamx.tranquility.druid.IndexService$$anonfun$call$1$$anonfun$apply$17.apply(IndexService.scala:132) at com.twitter.util.Future$$anonfun$map$1$$anonfun$apply$6.apply(Future.scala:950) at com.twitter.util.Try$.apply(Try.scala:13) at com.twitter.util.Future$.apply(Future.scala:97) at com.twitter.util.Future$$anonfun$map$1.apply(Future.scala:950) at com.twitter.util.Future$$anonfun$map$1.apply(Future.scala:949) at com.twitter.util.Promise$Transformer.liftedTree1$1(Promise.scala:112) at com.twitter.util.Promise$Transformer.k(Promise.scala:112) at com.twitter.util.Promise$Transformer.apply(Promise.scala:122) at com.twitter.util.Promise$Transformer.apply(Promise.scala:103) at com.twitter.util.Promise$$anon$1.run(Promise.scala:366) at com.twitter.concurrent.LocalScheduler$Activation.run(Scheduler.scala:178) at com.twitter.concurrent.LocalScheduler$Activation.submit(Scheduler.scala:136) at com.twitter.concurrent.LocalScheduler.submit(Scheduler.scala:207) at com.twitter.concurrent.Scheduler$.submit(Scheduler.scala:92) at com.twitter.util.Promise.runq(Promise.scala:350) at com.twitter.util.Promise.updateIfEmpty(Promise.scala:721) at com.twitter.util.Promise.update(Promise.scala:694) at com.twitter.util.Promise.setValue(Promise.scala:670) at com.twitter.concurrent.AsyncQueue.offer(AsyncQueue.scala:111) at com.twitter.finagle.netty3.transport.ChannelTransport.handleUpstream(ChannelTransport.scala:55) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.http.HttpContentDecoder.messageReceived(HttpContentDecoder.java:108) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:194) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459) at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536) at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.handler.codec.http.HttpClientCodec.handleUpstream(HttpClientCodec.java:92) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelHandler.messageReceived(SimpleChannelHandler.java:142) at com.twitter.finagle.netty3.channel.ChannelStatsHandler.messageReceived(ChannelStatsHandler.scala:78) at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:88) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelHandler.messageReceived(SimpleChannelHandler.java:142) at com.twitter.finagle.netty3.channel.ChannelRequestStatsHandler.messageReceived(ChannelRequestStatsHandler.scala:35) at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:88) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) ... 3 more

I don't know what is the problem?
Does my conf cluster is good?

Thanks in advance

sdesikan6 · Answer 1 · Sat Jul 22 2017 07:03:13 GMT+0800 (China Standard Time)

Hi You may want to post this in the Druid group -possibly a config issue on Druid. Thanks sd Sri Desikan, Co-Founder, Sparkline Data SNAP - The fastest datawarehouse powered by Apache Spark. <http://sparklinedata.com/> sdesikan@sparklinedata.com 805 242 6694 sparklinedata.com <http://sparklinedata.com/>

…

On Jul 19, 2017, at 1:32 AM, amoussoubaruch ***@***.***> wrote: Hello, I installed druid in cluster mode last week and now i'm trying to put kafaka data topic in one datasource. I'm using tranquility API for spark and i write a job spark to take data in kafka and push that in druid. So i have a cluster with 3 nodes 👍 Node 1 : overlord coordinator middleManager Node 2 : historical node middleManager brocker Node 3 : Like node 2 When i start my spark job i have this error java.lang.IllegalStateException: Failed to save new beam for identifier[druid:overlord/foo1107] timestamp[2017-07-19T08:00:00.000Z] at com.metamx.tranquility.beam.ClusteredBeam$$anonfun$2.applyOrElse(ClusteredBeam.scala:289) at com.metamx.tranquility.beam.ClusteredBeam$$anonfun$2.applyOrElse(ClusteredBeam.scala:286) at com.twitter.util.Future$$anonfun$rescue$1.apply(Future.scala:924) at com.twitter.util.Future$$anonfun$rescue$1.apply(Future.scala:922) at com.twitter.util.Promise$Transformer.liftedTree1$1(Promise.scala:112) at com.twitter.util.Promise$Transformer.k(Promise.scala:112) at com.twitter.util.Promise$Transformer.apply(Promise.scala:122) at com.twitter.util.Promise$Transformer.apply(Promise.scala:103) at com.twitter.util.Promise$$anon$1.run(Promise.scala:366) at com.twitter.concurrent.LocalScheduler$Activation.run(Scheduler.scala:178) at com.twitter.concurrent.LocalScheduler$Activation.submit(Scheduler.scala:136) at com.twitter.concurrent.LocalScheduler.submit(Scheduler.scala:207) at com.twitter.concurrent.Scheduler$.submit(Scheduler.scala:92) at com.twitter.util.Promise.runq(Promise.scala:350) at com.twitter.util.Promise.updateIfEmpty(Promise.scala:721) at com.twitter.util.ExecutorServiceFuturePool$$anon$2.run(FuturePool.scala:107) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: com.metamx.tranquility.druid.IndexServicePermanentException: Service[druid:overlord] call failed with status: 400 Bad Request at com.metamx.tranquility.druid.IndexService$$anonfun$call$1$$anonfun$apply$17.apply(IndexService.scala:158) at com.metamx.tranquility.druid.IndexService$$anonfun$call$1$$anonfun$apply$17.apply(IndexService.scala:132) at com.twitter.util.Future$$anonfun$map$1$$anonfun$apply$6.apply(Future.scala:950) at com.twitter.util.Try$.apply(Try.scala:13) at com.twitter.util.Future$.apply(Future.scala:97) at com.twitter.util.Future$$anonfun$map$1.apply(Future.scala:950) at com.twitter.util.Future$$anonfun$map$1.apply(Future.scala:949) at com.twitter.util.Promise$Transformer.liftedTree1$1(Promise.scala:112) at com.twitter.util.Promise$Transformer.k(Promise.scala:112) at com.twitter.util.Promise$Transformer.apply(Promise.scala:122) at com.twitter.util.Promise$Transformer.apply(Promise.scala:103) at com.twitter.util.Promise$$anon$1.run(Promise.scala:366) at com.twitter.concurrent.LocalScheduler$Activation.run(Scheduler.scala:178) at com.twitter.concurrent.LocalScheduler$Activation.submit(Scheduler.scala:136) at com.twitter.concurrent.LocalScheduler.submit(Scheduler.scala:207) at com.twitter.concurrent.Scheduler$.submit(Scheduler.scala:92) at com.twitter.util.Promise.runq(Promise.scala:350) at com.twitter.util.Promise.updateIfEmpty(Promise.scala:721) at com.twitter.util.Promise.update(Promise.scala:694) at com.twitter.util.Promise.setValue(Promise.scala:670) at com.twitter.concurrent.AsyncQueue.offer(AsyncQueue.scala:111) at com.twitter.finagle.netty3.transport.ChannelTransport.handleUpstream(ChannelTransport.scala:55) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.http.HttpContentDecoder.messageReceived(HttpContentDecoder.java:108) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.http.HttpChunkAggregator.messageReceived(HttpChunkAggregator.java:194) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296) at org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459) at org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536) at org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.handler.codec.http.HttpClientCodec.handleUpstream(HttpClientCodec.java:92) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelHandler.messageReceived(SimpleChannelHandler.java:142) at com.twitter.finagle.netty3.channel.ChannelStatsHandler.messageReceived(ChannelStatsHandler.scala:78) at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:88) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelHandler.messageReceived(SimpleChannelHandler.java:142) at com.twitter.finagle.netty3.channel.ChannelRequestStatsHandler.messageReceived(ChannelRequestStatsHandler.scala:35) at org.jboss.netty.channel.SimpleChannelHandler.handleUpstream(SimpleChannelHandler.java:88) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) ... 3 more I don't know what is the problem? Does my conf cluster is good? Thanks in advance — You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub <#69>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AHvYAt6NfsZOAPD2oPddF2amVbhlswN5ks5sPb8egaJpZM4OcZjj>.

weizhenyi · Answer 2 · Wed Oct 18 2017 11:14:22 GMT+0800 (China Standard Time)

please run /bin/init firstly!