elastic · mikemccand · Feb 25, 2017
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,6 +1,6 @@
 # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 elasticsearch     = 6.0.0-alpha1
-lucene            = 6.5.0-snapshot-f919485
+lucene            = 6.5.0-snapshot-d00c5ca
 
 # optional dependencies
 spatial4j         = 0.6

diff --git a/core/licenses/lucene-analyzers-common-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-analyzers-common-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+9ad2a7bd252cbdb76ac121287e670d75f4db2cd3
diff --git a/core/licenses/lucene-analyzers-common-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-analyzers-common-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-backward-codecs-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-backward-codecs-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+c6a940eff8a87df40262b752ed7b135e448b7873
diff --git a/core/licenses/lucene-backward-codecs-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-backward-codecs-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-core-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-core-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+6ef5ad88141760c00ea041da1535f3ffc364d67d
diff --git a/core/licenses/lucene-core-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-core-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-grouping-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-grouping-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+f15775571fb5762dfc92e00c3909cb8db8ff1d53
diff --git a/core/licenses/lucene-grouping-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-grouping-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-highlighter-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-highlighter-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+051d793aa64257beead4ccc7432eb5df81d17f23
diff --git a/core/licenses/lucene-highlighter-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-highlighter-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-join-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-join-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+5bc4cba55670c14ea812ff5de65edad4c312fdf6
diff --git a/core/licenses/lucene-join-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-join-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-memory-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-memory-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+68cf08bcd8414a57493debf3a6a509d78a9abb56
diff --git a/core/licenses/lucene-memory-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-memory-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-misc-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-misc-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+f5d90756dbeda1218d723b7bea0799c88d621adb
diff --git a/core/licenses/lucene-misc-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-misc-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-queries-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-queries-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+9298e7d1ed96e7beb63d7ccdce1a4502eb0fe484
diff --git a/core/licenses/lucene-queries-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-queries-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-queryparser-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-queryparser-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+918de18963607af69dff38e4773c0bde89c73ae3
diff --git a/core/licenses/lucene-queryparser-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-queryparser-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-sandbox-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-sandbox-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+a311a7d9f3e9a8fbf3a367a4e2731f9d4579732b
diff --git a/core/licenses/lucene-sandbox-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-sandbox-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-spatial-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-spatial-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+693bc4cb0e2e4465e0173c67ed0818071c4b460b
diff --git a/core/licenses/lucene-spatial-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-spatial-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-spatial-extras-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-spatial-extras-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+0326f31e63c76d476c23488c7354265cf915350f
diff --git a/core/licenses/lucene-spatial-extras-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-spatial-extras-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-spatial3d-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-spatial3d-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+69a3a86e9d045f872408793ea411d49e0c577268
diff --git a/core/licenses/lucene-spatial3d-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-spatial3d-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/licenses/lucene-suggest-6.5.0-snapshot-d00c5ca.jar.sha1 b/core/licenses/lucene-suggest-6.5.0-snapshot-d00c5ca.jar.sha1
@@ -0,0 +1 @@
+fabc05ca175150171cf60370877276b933716bcd
diff --git a/core/licenses/lucene-suggest-6.5.0-snapshot-f919485.jar.sha1 b/core/licenses/lucene-suggest-6.5.0-snapshot-f919485.jar.sha1
diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
@@ -33,7 +33,6 @@
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.GraphQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -747,6 +746,8 @@ private Query applySlop(Query q, int slop) {
             MultiPhraseQuery.Builder builder = new MultiPhraseQuery.Builder((MultiPhraseQuery) q);
             builder.setSlop(slop);
             return builder.build();
+            // nocommit
+            /*
         } else if (q instanceof GraphQuery && ((GraphQuery) q).hasPhrase()) {
             // we have a graph query that has at least one phrase sub-query
             // re-build and set slop on all phrase queries
@@ -762,6 +763,7 @@ private Query applySlop(Query q, int slop) {
             }
 
             return new GraphQuery(queries);
+            */
         } else {
             return q;
         }

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -262,7 +262,7 @@ public ScoreDoc[] sortDocs(boolean ignoreFrom, AtomicArray<? extends QuerySearch
                 // the 'index' field is the position in the resultsArr atomic array
                 shardTopDocs[sortedResult.index] = (TopFieldDocs) topDocs;
             }
-            mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs);
+            mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs, true);
         } else {
             final TopDocs[] shardTopDocs = new TopDocs[resultsArr.length()];
             if (result.size() != shardTopDocs.length) {
@@ -274,7 +274,7 @@ public ScoreDoc[] sortDocs(boolean ignoreFrom, AtomicArray<? extends QuerySearch
                 // the 'index' field is the position in the resultsArr atomic array
                 shardTopDocs[sortedResult.index] = topDocs;
             }
-            mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs);
+            mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs, true);
         }
 
         ScoreDoc[] scoreDocs = mergedTopDocs.scoreDocs;

diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java
@@ -18,17 +18,17 @@
  */
 package org.elasticsearch.index.analysis;
 
+import java.io.Reader;
+import java.util.regex.Pattern;
+
 import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 
-import java.io.Reader;
-import java.util.regex.Pattern;
-
-public class PatternReplaceCharFilterFactory extends AbstractCharFilterFactory {
+public class PatternReplaceCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent {
 
     private final Pattern pattern;
     private final String replacement;
@@ -56,4 +56,9 @@ public String getReplacement() {
     public Reader create(Reader tokenStream) {
         return new PatternReplaceCharFilter(pattern, replacement, tokenStream);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
@@ -27,7 +27,6 @@
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.GraphQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -318,6 +317,8 @@ protected Query newSynonymQuery(Term[] terms) {
 
         public Query createPhrasePrefixQuery(String field, String queryText, int phraseSlop, int maxExpansions) {
             final Query query = createFieldQuery(getAnalyzer(), Occur.MUST, field, queryText, true, phraseSlop);
+            // nocommit
+            /*
             if (query instanceof GraphQuery) {
                 // we have a graph query, convert inner queries to multi phrase prefix queries
                 List<Query> oldQueries = ((GraphQuery) query).getQueries();
@@ -328,6 +329,7 @@ public Query createPhrasePrefixQuery(String field, String queryText, int phraseS
 
                 return new GraphQuery(queries);
             }
+            */
 
             return toMultiPhrasePrefix(query, phraseSlop, maxExpansions);
         }

diff --git a/.../src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java b/.../src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java
@@ -119,15 +119,15 @@ public InternalAggregation doReduce(List<InternalAggregation> aggregations, Redu
                 shardDocs[i] = topHitsAgg.topDocs;
                 shardHits[i] = topHitsAgg.searchHits;
             }
-            reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs);
+            reducedTopDocs = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardDocs, true);
         } else {
             shardDocs = new TopDocs[aggregations.size()];
             for (int i = 0; i < shardDocs.length; i++) {
                 InternalTopHits topHitsAgg = (InternalTopHits) aggregations.get(i);
                 shardDocs[i] = topHitsAgg.topDocs;
                 shardHits[i] = topHitsAgg.searchHits;
             }
-            reducedTopDocs = TopDocs.merge(from, size, shardDocs);
+            reducedTopDocs = TopDocs.merge(from, size, shardDocs, true);
         }
 
         final int[] tracker = new int[shardHits.length];

diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
@@ -70,7 +70,7 @@ public Weight getWeight() {
     }
 
     @Override
-    public Collection<ChildScorer> getChildren() {
+    public Collection<ChildScorer> getChildren() throws IOException {
         return scorer.getChildren();
     }
 

diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
@@ -184,8 +184,10 @@ public SuggestDoc[] getResults() {
         private final SuggestDocPriorityQueue pq;
         private final Map<Integer, SuggestDoc> scoreDocMap;
 
+        // TODO: expose dup removal
+
         TopDocumentsCollector(int num) {
-            super(1); // TODO hack, we don't use the underlying pq, so we allocate a size of 1
+            super(1, false); // TODO hack, we don't use the underlying pq, so we allocate a size of 1
             this.num = num;
             this.scoreDocMap = new LinkedHashMap<>(num);
             this.pq = new SuggestDocPriorityQueue(num);

diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-6.5.0-snapshot-f919485.jar}" {
+grant codeBase "${codebase.lucene-core-6.5.0-snapshot-d00c5ca.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.5.0-snapshot-f919485.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.lucene-misc-6.5.0-snapshot-f919485.jar}" {
+grant codeBase "${codebase.lucene-misc-6.5.0-snapshot-d00c5ca.jar}" {
   // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
   permission java.nio.file.LinkPermission "hard";
 };

diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-6.5.0-snapshot-f919485.jar}" {
+grant codeBase "${codebase.lucene-test-framework-6.5.0-snapshot-d00c5ca.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed for testing hardlinks in StoreRecoveryTests since we install MockFS