elastic · jpountz · Jun 15, 2017 · Jun 14, 2017 · Jun 14, 2017 · Jun 14, 2017
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,6 +1,6 @@
 # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 elasticsearch     = 6.0.0-alpha3
-lucene            = 7.0.0-snapshot-a0aef2f
+lucene            = 7.0.0-snapshot-92b1783
 
 # optional dependencies
 spatial4j         = 0.6

diff --git a/core/licenses/lucene-analyzers-common-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+5bf8d8b7d885e25c343c187d1849580e21ef3fce
diff --git a/core/licenses/lucene-analyzers-common-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-analyzers-common-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-backward-codecs-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+9696b87e27ea949fabc62606833ab63e6e5473b9
diff --git a/core/licenses/lucene-backward-codecs-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-backward-codecs-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-core-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-core-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+d65b95dc24ce104e4d815b31f7159c5f6e97831d
diff --git a/core/licenses/lucene-core-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-core-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-grouping-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-grouping-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+3afa0db63adea8ee78b958cc85c5a6cb7750a5aa
diff --git a/core/licenses/lucene-grouping-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-grouping-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-highlighter-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-highlighter-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+586db5fba5b84d4955e349c3ca77b7df67498a24
diff --git a/core/licenses/lucene-highlighter-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-highlighter-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-join-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-join-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+8fc234d4474eaa400f5f964e18e9b179d87d86f0
diff --git a/core/licenses/lucene-join-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-join-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-memory-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-memory-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+3c70558114d053c025d04347b13bd10317c1db69
diff --git a/core/licenses/lucene-memory-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-memory-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-misc-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-misc-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+bf80c278e4c1c22b6e1382fc88ed016969596b61
diff --git a/core/licenses/lucene-misc-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-misc-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-queries-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-queries-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+fb2313a800903b991d21704ebcdce5f07a602259
diff --git a/core/licenses/lucene-queries-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-queries-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-queryparser-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-queryparser-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+24d1843ffaf4fddbd41c636274a9a8288ccdf964
diff --git a/core/licenses/lucene-queryparser-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-queryparser-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-sandbox-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-sandbox-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+6413231d34b23fcbca9fd17ea6c980b594e420ff
diff --git a/core/licenses/lucene-sandbox-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-sandbox-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-spatial-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-spatial-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+634187ab976bcde9905b4167ad273d3db6372a20
diff --git a/core/licenses/lucene-spatial-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-spatial-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-spatial-extras-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+c65576991cd1d9a75e6ee4e4a81e3d20bd160239
diff --git a/core/licenses/lucene-spatial-extras-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-spatial-extras-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-spatial3d-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-spatial3d-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+85c7a9adc02245b7a19e5cffed83cc20043cda83
diff --git a/core/licenses/lucene-spatial3d-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-spatial3d-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/licenses/lucene-suggest-7.0.0-snapshot-92b1783.jar.sha1 b/core/licenses/lucene-suggest-7.0.0-snapshot-92b1783.jar.sha1
@@ -0,0 +1 @@
+7ca7464c4b7900d7d514335d98c391851dcd84ce
diff --git a/core/licenses/lucene-suggest-7.0.0-snapshot-a0aef2f.jar.sha1 b/core/licenses/lucene-suggest-7.0.0-snapshot-a0aef2f.jar.sha1
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -24,6 +24,7 @@
 import org.apache.lucene.search.uhighlight.Snippet;
 import org.apache.lucene.search.uhighlight.BoundedBreakIteratorScanner;
 import org.apache.lucene.search.uhighlight.CustomPassageFormatter;
+import org.apache.lucene.search.uhighlight.CustomSeparatorBreakIterator;
 import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CollectionUtil;
@@ -96,9 +97,7 @@ public HighlightField highlight(HighlighterContext highlighterContext) {
                 // breaks the text on, so we don't lose the distinction between the different values of a field and we
                 // get back a snippet per value
                 String fieldValue = mergeFieldValues(fieldValues, MULTIVAL_SEP_CHAR);
-                org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator breakIterator =
-                    new org.apache.lucene.search.postingshighlight
-                        .CustomSeparatorBreakIterator(MULTIVAL_SEP_CHAR);
+                CustomSeparatorBreakIterator breakIterator = new CustomSeparatorBreakIterator(MULTIVAL_SEP_CHAR);
                 highlighter =
                     new CustomUnifiedHighlighter(searcher, analyzer, mapperHighlighterEntry.passageFormatter,
                         field.fieldOptions().boundaryScannerLocale(), breakIterator, fieldValue,

diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-7.0.0-snapshot-a0aef2f.jar}" {
+grant codeBase "${codebase.lucene-core-7.0.0-snapshot-92b1783.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-7.0.0-snapshot-a0aef2f.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.lucene-misc-7.0.0-snapshot-a0aef2f.jar}" {
+grant codeBase "${codebase.lucene-misc-7.0.0-snapshot-92b1783.jar}" {
   // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
   permission java.nio.file.LinkPermission "hard";
 };

diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-7.0.0-snapshot-a0aef2f.jar}" {
+grant codeBase "${codebase.lucene-test-framework-7.0.0-snapshot-92b1783.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed for testing hardlinks in StoreRecoveryTests since we install MockFS

diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
@@ -26,7 +26,9 @@
 import org.hamcrest.core.IsNull;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -280,14 +282,10 @@ public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception {
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubled"));
         String[] expectedAttributesKey = {
             "bytes",
+            "termFrequency",
             "positionLength"};
-        assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length));
-        Object extendedAttribute;
-
-        for (String key : expectedAttributesKey) {
-            extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key);
-            assertThat(extendedAttribute, notNullValue());
-        }
+        assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().keySet(),
+                equalTo(new HashSet<>(Arrays.asList(expectedAttributesKey))));
     }
 
     public void testDetailAnalyzeWithMultiValues() throws Exception {

diff --git a/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java b/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
@@ -126,8 +126,10 @@ public void testDfsQueryThenFetch() throws Exception {
         // to produce the same 8-bit norm for all docs here, so that
         // the tf is basically the entire score (assuming idf is fixed, which
         // it should be if dfs is working correctly)
-        for (int i = 1024; i < 1124; i++) {
-            index(Integer.toString(i - 1024), "test", i);
+        // With the current way of encoding norms, every length between 1048 and 1176
+        // are encoded into the same byte
+        for (int i = 1048; i < 1148; i++) {
+            index(Integer.toString(i - 1048), "test", i);
         }
         refresh();
 

diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
@@ -273,14 +273,14 @@ public void testBoostModeSettingWorks() throws Exception {
                 .setId("1")
                 .setIndex("test")
                 .setSource(
-                        jsonBuilder().startObject().field("test", "value").startObject("loc").field("lat", 11).field("lon", 21).endObject()
-                                .endObject()));
+                        jsonBuilder().startObject().field("test", "value value").startObject("loc").field("lat", 11).field("lon", 21)
+                                .endObject().endObject()));
         indexBuilders.add(client().prepareIndex()
                 .setType("type1")
                 .setId("2")
                 .setIndex("test")
                 .setSource(
-                        jsonBuilder().startObject().field("test", "value value").startObject("loc").field("lat", 11).field("lon", 20)
+                        jsonBuilder().startObject().field("test", "value").startObject("loc").field("lat", 11).field("lon", 20)
                                 .endObject().endObject()));
         indexRandom(true, false, indexBuilders); // force no dummy docs
 
@@ -297,10 +297,19 @@ public void testBoostModeSettingWorks() throws Exception {
         SearchResponse sr = response.actionGet();
         SearchHits sh = sr.getHits();
         assertThat(sh.getTotalHits(), equalTo((long) (2)));
-        assertThat(sh.getAt(0).getId(), isOneOf("1"));
+        assertThat(sh.getAt(0).getId(), equalTo("1"));
         assertThat(sh.getAt(1).getId(), equalTo("2"));
 
         // Test Exp
+        response = client().search(
+                searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source(
+                        searchSource().query(termQuery("test", "value"))));
+        sr = response.actionGet();
+        sh = sr.getHits();
+        assertThat(sh.getTotalHits(), equalTo((long) (2)));
+        assertThat(sh.getAt(0).getId(), equalTo("1"));
+        assertThat(sh.getAt(1).getId(), equalTo("2"));
+
         response = client().search(
                 searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source(
                         searchSource().query(

diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
@@ -294,8 +294,8 @@ public void testSmallRescoreWindow() throws Exception {
         assertThat(searchResponse.getHits().getHits().length, equalTo(4));
         assertHitCount(searchResponse, 4);
         assertThat(searchResponse.getHits().getMaxScore(), equalTo(searchResponse.getHits().getHits()[0].getScore()));
-        assertFirstHit(searchResponse, hasId("6"));
-        assertSecondHit(searchResponse, hasId("1"));
+        assertFirstHit(searchResponse, hasId("1"));
+        assertSecondHit(searchResponse, hasId("6"));
         assertThirdHit(searchResponse, hasId("3"));
         assertFourthHit(searchResponse, hasId("2"));
     }
@@ -392,29 +392,6 @@ private static void assertEquivalent(String query, SearchResponse plain, SearchR
         }
     }
 
-    private static void assertEquivalentOrSubstringMatch(String query, SearchResponse plain, SearchResponse rescored) {
-        assertNoFailures(plain);
-        assertNoFailures(rescored);
-        SearchHits leftHits = plain.getHits();
-        SearchHits rightHits = rescored.getHits();
-        assertThat(leftHits.getTotalHits(), equalTo(rightHits.getTotalHits()));
-        assertThat(leftHits.getHits().length, equalTo(rightHits.getHits().length));
-        SearchHit[] hits = leftHits.getHits();
-        SearchHit[] otherHits = rightHits.getHits();
-        if (!hits[0].getId().equals(otherHits[0].getId())) {
-            assertThat(((String) otherHits[0].getSourceAsMap().get("field1")).contains(query), equalTo(true));
-        } else {
-            Arrays.sort(hits, searchHitsComparator);
-            Arrays.sort(otherHits, searchHitsComparator);
-            for (int i = 0; i < hits.length; i++) {
-                if (hits[i].getScore() == hits[hits.length-1].getScore()) {
-                    return; // we need to cut off here since this is the tail of the queue and we might not have fetched enough docs
-                }
-                assertThat(query, hits[i].getId(), equalTo(rightHits.getHits()[i].getId()));
-            }
-        }
-    }
-
     // forces QUERY_THEN_FETCH because of https://github.com/elastic/elasticsearch/issues/4829
     public void testEquivalence() throws Exception {
         // no dummy docs since merges can change scores while we run queries.
@@ -461,18 +438,6 @@ public void testEquivalence() throws Exception {
                     .actionGet();
             // check equivalence
             assertEquivalent(query, plain, rescored);
-
-            rescored = client()
-                    .prepareSearch()
-                    .setSearchType(SearchType.QUERY_THEN_FETCH)
-                    .setPreference("test") // ensure we hit the same shards for tie-breaking
-                    .setQuery(QueryBuilders.matchQuery("field1", query).operator(Operator.OR))
-                    .setFrom(0)
-                    .setSize(resultSize)
-                    .setRescorer(queryRescorer(matchPhraseQuery("field1", intToEnglish).slop(0))
-                                    .setQueryWeight(1.0f).setRescoreQueryWeight(1.0f), 2 * rescoreWindow).execute().actionGet();
-            // check equivalence or if the first match differs we check if the phrase is a substring of the top doc
-            assertEquivalentOrSubstringMatch(intToEnglish, plain, rescored);
         }
     }
 

diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
@@ -241,13 +241,13 @@ The output from the above is:
   },
   "hits": {
     "total": 1,
-    "max_score": 0.2824934,
+    "max_score": 0.2876821,
     "hits": [
       {
         "_index": "my_index",
         "_type": "my_type",
         "_id": "1",
-        "_score": 0.2824934,
+        "_score": 0.2876821,
         "_source": {
           "text": "The fooBarBaz method"
         },

diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
@@ -300,13 +300,13 @@ GET my_index/_search
   },
   "hits": {
     "total": 1,
-    "max_score": 0.51623213,
+    "max_score": 0.5753642,
     "hits": [
       {
         "_index": "my_index",
         "_type": "doc",
         "_id": "1",
-        "_score": 0.51623213,
+        "_score": 0.5753642,
         "_source": {
           "title": "Quick Foxes"
         }

diff --git a/docs/reference/how-to/recipes.asciidoc b/docs/reference/how-to/recipes.asciidoc
@@ -88,13 +88,13 @@ GET index/_search
   },
   "hits": {
     "total": 2,
-    "max_score": 0.25811607,
+    "max_score": 0.2876821,
     "hits": [
       {
         "_index": "index",
         "_type": "type",
         "_id": "2",
-        "_score": 0.25811607,
+        "_score": 0.2876821,
         "_source": {
           "body": "A pair of skis"
         }
@@ -103,7 +103,7 @@ GET index/_search
         "_index": "index",
         "_type": "type",
         "_id": "1",
-        "_score": 0.25811607,
+        "_score": 0.2876821,
         "_source": {
           "body": "Ski resort"
         }
@@ -145,13 +145,13 @@ GET index/_search
   },
   "hits": {
     "total": 1,
-    "max_score": 0.25811607,
+    "max_score": 0.2876821,
     "hits": [
       {
         "_index": "index",
         "_type": "type",
         "_id": "1",
-        "_score": 0.25811607,
+        "_score": 0.2876821,
         "_source": {
           "body": "Ski resort"
         }
@@ -201,13 +201,13 @@ GET index/_search
   },
   "hits": {
     "total": 1,
-    "max_score": 0.25811607,
+    "max_score": 0.2876821,
     "hits": [
       {
         "_index": "index",
         "_type": "type",
         "_id": "1",
-        "_score": 0.25811607,
+        "_score": 0.2876821,
         "_source": {
           "body": "Ski resort"
         }