Lucene error - IndexOutOfBoundsException accessing norms #27131
Description
Came across this bug today on master and I have been unable to reduce to a simple case.
Lucene throws an exception accessing norms only when given the following conditions:
- Only when query is for term A OR term B (doesn't fail with either of the terms independently)
- Only when using top docs (works OK with hits)
- Only when the docs matching the query sit in an index with other docs.
The gist with the data and queries to reproduce this is here:
https://gist.github.com/markharwood/3bdf60db7887ec5eb7a0a8fd7074cfae
I have reduced the number of docs ingested to the smallest set of fields and the smallest number of records I could before the error is revealed.
Error thrown is
Caused by: java.lang.IndexOutOfBoundsException: 2147483647
at java.nio.DirectByteBuffer.get(DirectByteBuffer.java:253) ~[?:1.8.0_144]
at org.apache.lucene.store.ByteBufferGuard.getByte(ByteBufferGuard.java:118) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.store.ByteBufferIndexInput$SingleBufferImpl.readByte(ByteBufferIndexInput.java:385) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.codecs.lucene70.Lucene70NormsProducer$2.longValue(Lucene70NormsProducer.java:218) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.similarities.BM25Similarity$BM25DocScorer.score(BM25Similarity.java:253) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.TermScorer.score(TermScorer.java:66) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.TopScoreDocCollector$SimpleTopScoreDocCollector$1.collect(TopScoreDocCollector.java:64) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregator$1.collect(TopHitsAggregator.java:132) ~[1/:?]
at org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:82) ~[1/:?]
at org.apache.lucene.search.MultiCollector$MultiLeafCollector.collect(MultiCollector.java:174) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.BooleanScorer.scoreDocument(BooleanScorer.java:189) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.BooleanScorer.scoreMatches(BooleanScorer.java:202) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.BooleanScorer.scoreWindowIntoBitSetAndReplay(BooleanScorer.java:216) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.BooleanScorer.scoreWindowMultipleScorers(BooleanScorer.java:260) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:305) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:317) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:658) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:186) ~[1/:?]
at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462) ~[lucene-core-7.1.0-snapshot-f33ed4ba12a.jar:7.1.0-snapshot-f33ed4ba12a f33ed4ba12aaf215628d010daaa0e271b8ab3d1f - mvg - 2017-10-02 17:18:30]
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:272) ~[1/:?]
at org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:110) ~[1/:?]
at org.elasticsearch.indices.IndicesService.lambda$16(IndicesService.java:1122) ~[1/:?]
at org.elasticsearch.indices.IndicesService.lambda$17(IndicesService.java:1175) ~[1/:?]
at org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:160) ~[1/:?]
at org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:1) ~[1/:?]
at org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:412) ~[1/:?]
at org.elasticsearch.indices.IndicesRequestCache.getOrCompute(IndicesRequestCache.java:116) ~[1/:?]
at org.elasticsearch.indices.IndicesService.cacheShardLevelResult(IndicesService.java:1181) ~[1/:?]
at org.elasticsearch.indices.IndicesService.loadIntoContext(IndicesService.java:1121) ~[1/:?]
at org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:300) ~[1/:?]
at org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:335) ~[1/:?]
at org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:311) ~[1/:?]