Netty4HttpServerPipeliningTests fails on 6.4 due to invalid ref count decrement #32900
Description
Build URL: http://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+6.4+intake/60
Reproduce commands:
./gradlew :modules:transport-netty4:test \
-Dtests.seed=FB3A869C6124570A \
-Dtests.class=org.elasticsearch.http.netty4.Netty4HttpServerPipeliningTests \
-Dtests.method="testThatHttpPipeliningCanBeDisabled" \
-Dtests.security.manager=true \
-Dtests.locale=sl \
-Dtests.timezone=Asia/Magadan
./gradlew :modules:transport-netty4:test \
-Dtests.seed=FB3A869C6124570A \
-Dtests.class=org.elasticsearch.http.netty4.Netty4HttpServerPipeliningTests \
-Dtests.method="testThatHttpPipeliningCanBeDisabled" \
-Dtests.security.manager=true \
-Dtests.locale=sl \
-Dtests.timezone=Asia/Magadan
This doesn't reproduce for me locally but it seems that there is a bug where we are trying to decrement a reference counter when its already zero:
1> [2018-08-16T03:19:43,021][INFO ][o.e.h.n.Netty4HttpServerPipeliningTests] Caught exception
16:20:19 1> io.netty.util.IllegalReferenceCountException: refCnt: 0, decrement: 1
16:20:19 1> at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:102) ~[netty-buffer-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:90) ~[netty-buffer-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.buffer.WrappedCompositeByteBuf.release(WrappedCompositeByteBuf.java:43) ~[netty-buffer-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.buffer.SimpleLeakAwareCompositeByteBuf.release(SimpleLeakAwareCompositeByteBuf.java:38) ~[netty-buffer-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.buffer.AdvancedLeakAwareCompositeByteBuf.release(AdvancedLeakAwareCompositeByteBuf.java:1023) ~[netty-buffer-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.handler.codec.http.HttpObjectAggregator$AggregatedFullHttpMessage.release(HttpObjectAggregator.java:386) ~[netty-codec-http-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.util.ReferenceCountUtil.release(ReferenceCountUtil.java:84) ~[netty-common-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:112) ~[netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.16.Final.jar:4.1.16.Final]
16:20:19 1> at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.16.Final.jar:4.1.16.Final]