Adding org.apache.spark:spark-core_2.12:2.4.0 into dependencies causes every request to throw "ERROR i.m.h.s.netty.RoutingInBoundHandler - Unexpected error occurred: refCnt: 0, increment: 1"
make opened this issue · comments
Steps to Reproduce
- Add following into dependencies:
compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7'
compile 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7'
compile 'org.apache.spark:spark-core_2.12:2.4.0'
- curl any HTTP endpoint in app
- The log shows:
14:50:43.088 [nioEventLoopGroup-1-2] ERROR i.m.h.s.netty.RoutingInBoundHandler - Unexpected error occurred: refCnt: 0, increment: 1
io.netty.util.IllegalReferenceCountException: refCnt: 0, increment: 1
at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:100)
at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:84)
at io.netty.handler.codec.http.DefaultFullHttpResponse.release(DefaultFullHttpResponse.java:115)
at io.netty.util.ReferenceCountUtil.release(ReferenceCountUtil.java:88)
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:720)
at io.netty.handler.stream.ChunkedWriteHandler.doFlush(ChunkedWriteHandler.java:304)
at io.netty.handler.stream.ChunkedWriteHandler.channelWritabilityChanged(ChunkedWriteHandler.java:150)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:434)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:416)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelWritabilityChanged(AbstractChannelHandlerContext.java:409)
at io.micronaut.http.netty.stream.HttpStreamsHandler.flushNext(HttpStreamsHandler.java:397)
at io.micronaut.http.netty.stream.HttpStreamsHandler.lambda$null$0(HttpStreamsHandler.java:318)
at io.micronaut.http.netty.stream.HttpStreamsHandler.executeInEventLoop(HttpStreamsHandler.java:403)
at io.micronaut.http.netty.stream.HttpStreamsHandler.lambda$unbufferedWrite$1(HttpStreamsHandler.java:315)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:481)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420)
at io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:163)
at io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:93)
at io.micronaut.http.netty.stream.HttpStreamsHandler.unbufferedWrite(HttpStreamsHandler.java:315)
at io.micronaut.http.netty.stream.HttpStreamsServerHandler.unbufferedWrite(HttpStreamsServerHandler.java:178)
at io.micronaut.http.netty.stream.HttpStreamsHandler.flushNext(HttpStreamsHandler.java:395)
at io.micronaut.http.netty.stream.HttpStreamsHandler.write(HttpStreamsHandler.java:291)
at io.micronaut.http.netty.stream.HttpStreamsServerHandler.write(HttpStreamsServerHandler.java:58)
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:738)
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:730)
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:816)
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:723)
at io.netty.handler.stream.ChunkedWriteHandler.doFlush(ChunkedWriteHandler.java:304)
at io.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:137)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768)
at io.netty.channel.AbstractChannelHandlerContext.access$1500(AbstractChannelHandlerContext.java:38)
at io.netty.channel.AbstractChannelHandlerContext$WriteAndFlushTask.write(AbstractChannelHandlerContext.java:1129)
at io.netty.channel.AbstractChannelHandlerContext$AbstractWriteTask.run(AbstractChannelHandlerContext.java:1070)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:403)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:463)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
at java.lang.Thread.run(Thread.java:748)
14:50:43.124 [nioEventLoopGroup-1-2] WARN i.n.c.AbstractChannelHandlerContext - An exception was thrown by a user handler while handling an exceptionCaught event
io.netty.util.IllegalReferenceCountException: refCnt: 0, increment: 1
at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:100)
at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:84)
at io.netty.handler.codec.http.DefaultFullHttpResponse.release(DefaultFullHttpResponse.java:115)
at io.netty.util.ReferenceCountUtil.release(ReferenceCountUtil.java:88)
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:720)
at io.netty.handler.stream.ChunkedWriteHandler.doFlush(ChunkedWriteHandler.java:304)
at io.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:137)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768)
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749)
at io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776)
at io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:802)
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:814)
at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:794)
at io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:831)
at io.micronaut.http.server.netty.RoutingInBoundHandler.writeFinalNettyResponse(RoutingInBoundHandler.java:1134)
at io.micronaut.http.server.netty.RoutingInBoundHandler.access$500(RoutingInBoundHandler.java:129)
at io.micronaut.http.server.netty.RoutingInBoundHandler$3.onComplete(RoutingInBoundHandler.java:1095)
at io.micronaut.http.server.netty.RoutingInBoundHandler$3.onComplete(RoutingInBoundHandler.java:1092)
at io.micronaut.http.server.netty.async.ContextCompletionAwareSubscriber.doOnComplete(ContextCompletionAwareSubscriber.java:74)
at io.micronaut.core.async.subscriber.CompletionAwareSubscriber.onComplete(CompletionAwareSubscriber.java:72)
at io.reactivex.internal.util.HalfSerializer.onComplete(HalfSerializer.java:90)
at io.reactivex.internal.subscribers.StrictSubscriber.onComplete(StrictSubscriber.java:109)
at io.reactivex.internal.subscribers.BasicFuseableSubscriber.onComplete(BasicFuseableSubscriber.java:120)
at io.reactivex.internal.subscriptions.ScalarSubscription.request(ScalarSubscription.java:57)
at io.reactivex.internal.subscribers.BasicFuseableSubscriber.request(BasicFuseableSubscriber.java:153)
at io.reactivex.internal.subscriptions.SubscriptionHelper.deferredSetOnce(SubscriptionHelper.java:210)
at io.reactivex.internal.subscribers.StrictSubscriber.onSubscribe(StrictSubscriber.java:87)
at io.reactivex.internal.subscribers.BasicFuseableSubscriber.onSubscribe(BasicFuseableSubscriber.java:67)
at io.reactivex.internal.operators.flowable.FlowableJust.subscribeActual(FlowableJust.java:34)
at io.reactivex.Flowable.subscribe(Flowable.java:14479)
at io.reactivex.internal.operators.flowable.FlowableMap.subscribeActual(FlowableMap.java:37)
at io.reactivex.Flowable.subscribe(Flowable.java:14479)
at io.reactivex.Flowable.subscribe(Flowable.java:14429)
at io.micronaut.http.server.netty.RoutingInBoundHandler.subscribeToResponsePublisher(RoutingInBoundHandler.java:1092)
at io.micronaut.http.server.netty.RoutingInBoundHandler.writeDefaultErrorResponse(RoutingInBoundHandler.java:1445)
at io.micronaut.http.server.netty.RoutingInBoundHandler.exceptionCaught(RoutingInBoundHandler.java:339)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:285)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:264)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:256)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:285)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:264)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:256)
at io.netty.channel.ChannelHandlerAdapter.exceptionCaught(ChannelHandlerAdapter.java:87)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:285)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:264)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:256)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:285)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:264)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:256)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:131)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:285)
at io.netty.channel.AbstractChannelHandlerContext.notifyHandlerException(AbstractChannelHandlerContext.java:850)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:436)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelWritabilityChanged(AbstractChannelHandlerContext.java:416)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelWritabilityChanged(AbstractChannelHandlerContext.java:409)
at io.micronaut.http.netty.stream.HttpStreamsHandler.flushNext(HttpStreamsHandler.java:397)
at io.micronaut.http.netty.stream.HttpStreamsHandler.lambda$null$0(HttpStreamsHandler.java:318)
at io.micronaut.http.netty.stream.HttpStreamsHandler.executeInEventLoop(HttpStreamsHandler.java:403)
at io.micronaut.http.netty.stream.HttpStreamsHandler.lambda$unbufferedWrite$1(HttpStreamsHandler.java:315)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:481)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420)
at io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:163)
at io.netty.channel.DefaultChannelPromise.addListener(DefaultChannelPromise.java:93)
at io.micronaut.http.netty.stream.HttpStreamsHandler.unbufferedWrite(HttpStreamsHandler.java:315)
at io.micronaut.http.netty.stream.HttpStreamsServerHandler.unbufferedWrite(HttpStreamsServerHandler.java:178)
at io.micronaut.http.netty.stream.HttpStreamsHandler.flushNext(HttpStreamsHandler.java:395)
at io.micronaut.http.netty.stream.HttpStreamsHandler.write(HttpStreamsHandler.java:291)
at io.micronaut.http.netty.stream.HttpStreamsServerHandler.write(HttpStreamsServerHandler.java:58)
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:738)
at io.netty.channel.AbstractChannelHandlerContext.invokeWrite(AbstractChannelHandlerContext.java:730)
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:816)
at io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:723)
at io.netty.handler.stream.ChunkedWriteHandler.doFlush(ChunkedWriteHandler.java:304)
at io.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:137)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776)
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768)
at io.netty.channel.AbstractChannelHandlerContext.access$1500(AbstractChannelHandlerContext.java:38)
at io.netty.channel.AbstractChannelHandlerContext$WriteAndFlushTask.write(AbstractChannelHandlerContext.java:1129)
at io.netty.channel.AbstractChannelHandlerContext$AbstractWriteTask.run(AbstractChannelHandlerContext.java:1070)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:403)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:463)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
at java.lang.Thread.run(Thread.java:748)
Expected Behaviour
No errors should appear to the log.
Actual Behaviour
The error above appears to the log.
Environment Information
- Operating System: Ubuntu 16.04.5 LTS
- Micronaut Version: 1.0.0
- JDK Version: 1.8.0_191
Example Application
Will take a look, but why are you placing an older version of Jackson on the class path?
If not placing the older Jackson version following error appears to the log :
15:55:51.635 [main] ERROR io.micronaut.runtime.Micronaut - Error starting Micronaut server: Failed to inject value for parameter [objectMapper] of class: io.micronaut.jackson.convert.ArrayNodeToArrayConverter
Path Taken: new ArrayNodeToArrayConverter([ObjectMapper objectMapper])
io.micronaut.context.exceptions.DependencyInjectionException: Failed to inject value for parameter [objectMapper] of class: io.micronaut.jackson.convert.ArrayNodeToArrayConverter
Path Taken: new ArrayNodeToArrayConverter([ObjectMapper objectMapper])
at io.micronaut.context.AbstractBeanDefinition.getBeanForConstructorArgument(AbstractBeanDefinition.java:981)
at io.micronaut.jackson.convert.$ArrayNodeToArrayConverterDefinition.build(Unknown Source)
at io.micronaut.context.DefaultBeanContext.doCreateBean(DefaultBeanContext.java:1310)
at io.micronaut.context.DefaultBeanContext.addCandidateToList(DefaultBeanContext.java:2200)
at io.micronaut.context.DefaultBeanContext.getBeansOfTypeInternal(DefaultBeanContext.java:2131)
at io.micronaut.context.DefaultBeanContext.getBeansOfType(DefaultBeanContext.java:752)
at io.micronaut.context.DefaultBeanContext.getBeansOfType(DefaultBeanContext.java:522)
at io.micronaut.context.DefaultBeanContext.getBeanRegistrations(DefaultBeanContext.java:313)
at io.micronaut.context.DefaultApplicationContext.initializeTypeConverters(DefaultApplicationContext.java:330)
at io.micronaut.context.DefaultApplicationContext.initializeContext(DefaultApplicationContext.java:184)
at io.micronaut.context.DefaultBeanContext.readAllBeanDefinitionClasses(DefaultBeanContext.java:1999)
at io.micronaut.context.DefaultBeanContext.start(DefaultBeanContext.java:156)
at io.micronaut.context.DefaultApplicationContext.start(DefaultApplicationContext.java:138)
at io.micronaut.runtime.Micronaut.start(Micronaut.java:67)
at io.micronaut.runtime.Micronaut.run(Micronaut.java:271)
at io.micronaut.runtime.Micronaut.run(Micronaut.java:257)
at com.example.Application.main(Application.java:8)
Caused by: io.micronaut.context.exceptions.BeanInstantiationException: Error instantiating bean of type [io.micronaut.jackson.convert.ArrayNodeToArrayConverter]
Message: Incompatible Jackson version: 2.9.7
Path Taken: new ArrayNodeToArrayConverter([ObjectMapper objectMapper])
at io.micronaut.context.DefaultBeanContext.doCreateBean(DefaultBeanContext.java:1324)
at io.micronaut.context.DefaultBeanContext.createAndRegisterSingleton(DefaultBeanContext.java:1868)
at io.micronaut.context.DefaultBeanContext.getBeanForDefinition(DefaultBeanContext.java:1604)
at io.micronaut.context.DefaultBeanContext.getBeanInternal(DefaultBeanContext.java:1584)
at io.micronaut.context.DefaultBeanContext.getBean(DefaultBeanContext.java:884)
at io.micronaut.context.AbstractBeanDefinition.getBeanForConstructorArgument(AbstractBeanDefinition.java:973)
... 16 common frames omitted
Caused by: com.fasterxml.jackson.databind.JsonMappingException: Incompatible Jackson version: 2.9.7
at com.fasterxml.jackson.module.scala.JacksonModule.setupModule(JacksonModule.scala:64)
at com.fasterxml.jackson.module.scala.JacksonModule.setupModule$(JacksonModule.scala:51)
at com.fasterxml.jackson.module.scala.DefaultScalaModule.setupModule(DefaultScalaModule.scala:19)
at com.fasterxml.jackson.databind.ObjectMapper.registerModule(ObjectMapper.java:751)
at com.fasterxml.jackson.databind.ObjectMapper.registerModules(ObjectMapper.java:951)
at com.fasterxml.jackson.databind.ObjectMapper.findAndRegisterModules(ObjectMapper.java:1028)
at io.micronaut.jackson.ObjectMapperFactory.objectMapper(ObjectMapperFactory.java:86)
at io.micronaut.jackson.$ObjectMapperFactory$ObjectMapperDefinition.build(Unknown Source)
at io.micronaut.context.DefaultBeanContext.doCreateBean(DefaultBeanContext.java:1310)
... 21 common frames omitted
FAILURE: Build failed with an exception.
Well, I found that instead of placing older Jackson we can just add compile 'com.fasterxml.jackson.module:jackson-module-scala_2.12:2.9.7' into dependencies.
But this doesn't fix the error.
Looks like a netty conflict. Adding this causes the problem to go away
compile('org.apache.spark:spark-core_2.12:2.4.0') {
exclude group: "io.netty", module: "netty-all"
}
Let me know if this solution will work for you
Yes, that fixes the problem. Thanks!