Cumulative 7.x backport

docs/reference/cat/allocation.asciidoc

@@ -21,5 +21,6 @@ shards disk.indices disk.used disk.avail disk.total disk.percent host      ip
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/ s/46/\\d+/]
 // TESTRESPONSE[s/CSUXak2/.+/ _cat]
 
-Here we can see that each node has been allocated a single shard and
-that they're all using about the same amount of space.
+Here we can see that the single shard created has been allocated to the single
+node available.
+

docs/reference/query-dsl/dis-max-query.asciidoc

@@ -20,7 +20,7 @@ of these DisjunctionMaxQuery's is combined into a BooleanQuery.
 The tie breaker capability allows results that include the same term in
 multiple fields to be judged better than results that include this term
 in only the best of those multiple fields, without confusing this with
-the better case of two different terms in the multiple fields.The
+the better case of two different terms in the multiple fields. The
 default `tie_breaker` is `0.0`.
 
 This query maps to Lucene `DisjunctionMaxQuery`.

docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java

@@ -19,11 +19,12 @@
 
 package org.elasticsearch.smoketest;
 
-import org.apache.http.HttpHost;
-import org.apache.lucene.util.BytesRef;
-
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.http.HttpHost;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.ParseField;
@@ -48,12 +49,13 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 
+//The default 20 minutes timeout isn't always enough, please do not increase further than 30 before analyzing what makes this suite so slow
+@TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
 public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public DocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

qa/multi-cluster-search/build.gradle

@@ -21,6 +21,10 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
 
 apply plugin: 'elasticsearch.standalone-test'
 
+dependencies {
+  testCompile "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}"
+}
+
 task remoteClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
 }
@@ -53,6 +57,6 @@ task integTest {
   dependsOn = [mixedClusterTest]
 }
 
-unitTest.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
+unitTest.enabled = false // no unit tests for multi-cluster-search, only integration tests
 
 check.dependsOn(integTest)

qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java → qa/multi-cluster-search/src/test/java/org/elasticsearch/search/MultiClusterSearchYamlTestSuiteIT.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.upgrades;
+package org.elasticsearch.search;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@@ -42,5 +42,4 @@ public MultiClusterSearchYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t
     public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
-
 }

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -486,7 +486,7 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
         }
         ReduceContext reduceContext = reduceContextFunction.apply(performFinalReduce);
         final InternalAggregations aggregations = aggregationsList.isEmpty() ? null :
-            InternalAggregations.reduce(aggregationsList, firstResult.pipelineAggregators(), reduceContext);
+            InternalAggregations.reduce(aggregationsList, reduceContext);
         final SearchProfileShardResults shardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
         final SortedTopDocs sortedTopDocs = sortDocs(isScrollRequest, queryResults, bufferedTopDocs, topDocsStats, from, size,
             reducedCompletionSuggestions);

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -157,10 +157,19 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
         for (ExpressionResolver expressionResolver : expressionResolvers) {
             expressions = expressionResolver.resolve(context, expressions);
         }
-
+        
         if (expressions.isEmpty()) {
             if (!options.allowNoIndices()) {
-                IndexNotFoundException infe = new IndexNotFoundException((String)null);
+                IndexNotFoundException infe;
+                if (indexExpressions.length == 1) {
+                    if (indexExpressions[0].equals(MetaData.ALL)) {
+                        infe = new IndexNotFoundException("no indices exist", (String)null);
+                    } else {
+                        infe = new IndexNotFoundException((String)null);
+                    }
+                } else {
+                    infe = new IndexNotFoundException((String)null);
+                }
                 infe.setResources("index_expression", indexExpressions);
                 throw infe;
             } else {
@@ -173,7 +182,12 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
             AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(expression);
             if (aliasOrIndex == null ) {
                 if (failNoIndices) {
-                    IndexNotFoundException infe = new IndexNotFoundException(expression);
+                    IndexNotFoundException infe;
+                    if (expression.equals(MetaData.ALL)) {
+                        infe = new IndexNotFoundException("no indices exist", expression);
+                    } else {
+                        infe = new IndexNotFoundException(expression);  
+                    }
                     infe.setResources("index_expression", expression);
                     throw infe;
                 } else {

server/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java

@@ -132,23 +132,18 @@ public void execute(SearchContext context) {
                 throw new AggregationExecutionException("Failed to build aggregation [" + aggregator.name() + "]", e);
             }
         }
-        context.queryResult().aggregations(new InternalAggregations(aggregations));
-        try {
-            List<PipelineAggregator> pipelineAggregators = context.aggregations().factories().createPipelineAggregators();
-            List<SiblingPipelineAggregator> siblingPipelineAggregators = new ArrayList<>(pipelineAggregators.size());
-            for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
-                if (pipelineAggregator instanceof SiblingPipelineAggregator) {
-                    siblingPipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator);
-                } else {
-                    throw new AggregationExecutionException("Invalid pipeline aggregation named [" + pipelineAggregator.name()
-                            + "] of type [" + pipelineAggregator.getWriteableName() + "]. Only sibling pipeline aggregations are "
-                            + "allowed at the top level");
-                }
+        List<PipelineAggregator> pipelineAggregators = context.aggregations().factories().createPipelineAggregators();
+        List<SiblingPipelineAggregator> siblingPipelineAggregators = new ArrayList<>(pipelineAggregators.size());
+        for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
+            if (pipelineAggregator instanceof SiblingPipelineAggregator) {
+                siblingPipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator);
+            } else {
+                throw new AggregationExecutionException("Invalid pipeline aggregation named [" + pipelineAggregator.name()
+                    + "] of type [" + pipelineAggregator.getWriteableName() + "]. Only sibling pipeline aggregations are "
+                    + "allowed at the top level");
             }
-            context.queryResult().pipelineAggregators(siblingPipelineAggregators);
-        } catch (IOException e) {
-            throw new AggregationExecutionException("Failed to build top level pipeline aggregators", e);
         }
+        context.queryResult().aggregations(new InternalAggregations(aggregations, siblingPipelineAggregators));
 
         // disable aggregations so that they don't run on next pages in case of scrolling
         context.aggregations(null);

server/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java

@@ -175,7 +175,7 @@ private AggregatorFactories(AggregatorFactory<?>[] factories, List<PipelineAggre
         this.pipelineAggregatorFactories = pipelineAggregators;
     }
 
-    public List<PipelineAggregator> createPipelineAggregators() throws IOException {
+    public List<PipelineAggregator> createPipelineAggregators() {
         List<PipelineAggregator> pipelineAggregators = new ArrayList<>(this.pipelineAggregatorFactories.size());
         for (PipelineAggregationBuilder factory : this.pipelineAggregatorFactories) {
             pipelineAggregators.add(factory.create());

server/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java

@@ -78,7 +78,7 @@ public InternalAggregations(List<InternalAggregation> aggregations, List<Sibling
      * Note that top-level pipeline aggregators become normal aggregation once the final reduction has been performed, after which they
      * become part of the list of {@link InternalAggregation}s.
      */
-    List<SiblingPipelineAggregator> getTopLevelPipelineAggregators() {
+    public List<SiblingPipelineAggregator> getTopLevelPipelineAggregators() {
         return topLevelPipelineAggregators;
     }
 
@@ -92,20 +92,7 @@ public static InternalAggregations reduce(List<InternalAggregations> aggregation
         if (aggregationsList.isEmpty()) {
             return null;
         }
-        InternalAggregations first = aggregationsList.get(0);
-        return reduce(aggregationsList, first.topLevelPipelineAggregators, context);
-    }
-
-    /**
-     * Reduces the given list of aggregations as well as the provided top-level pipeline aggregators.
-     * Note that top-level pipeline aggregators are reduced only as part of the final reduction phase, otherwise they are left untouched.
-     */
-    public static InternalAggregations reduce(List<InternalAggregations> aggregationsList,
-                                              List<SiblingPipelineAggregator> topLevelPipelineAggregators,
-                                              ReduceContext context) {
-        if (aggregationsList.isEmpty()) {
-            return null;
-        }
+        List<SiblingPipelineAggregator> topLevelPipelineAggregators = aggregationsList.get(0).getTopLevelPipelineAggregators();
 
         // first we collect all aggregations of the same type and list them together
         Map<String, List<InternalAggregation>> aggByName = new HashMap<>();

server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java

@@ -24,7 +24,6 @@
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
-import java.io.IOException;
 import java.util.Collection;
 import java.util.Map;
 
@@ -76,7 +75,7 @@ protected abstract void validate(AggregatorFactory<?> parent, Collection<Aggrega
      *
      * @return The created aggregator
      */
-    protected abstract PipelineAggregator create() throws IOException;
+    protected abstract PipelineAggregator create();
 
     /** Associate metadata with this {@link PipelineAggregationBuilder}. */
     @Override

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java

@@ -89,15 +89,15 @@ public final void validate(AggregatorFactory<?> parent, Collection<AggregationBu
         doValidate(parent, factories, pipelineAggregatorFactories);
     }
 
-    protected abstract PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException;
+    protected abstract PipelineAggregator createInternal(Map<String, Object> metaData);
 
     /**
      * Creates the pipeline aggregator
      *
      * @return The created aggregator
      */
     @Override
-    public final PipelineAggregator create() throws IOException {
+    public final PipelineAggregator create() {
         PipelineAggregator aggregator = createInternal(this.metaData);
         return aggregator;
     }

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/AvgBucketPipelineAggregationBuilder.java

@@ -46,7 +46,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new AvgBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketMetricsPipelineAggregationBuilder.java

@@ -104,7 +104,7 @@ public GapPolicy gapPolicy() {
     }
 
     @Override
-    protected abstract PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException;
+    protected abstract PipelineAggregator createInternal(Map<String, Object> metaData);
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilder> aggBuilders,

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptPipelineAggregationBuilder.java

@@ -139,7 +139,7 @@ public GapPolicy gapPolicy() {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new BucketScriptPipelineAggregator(name, bucketsPathsMap, script, formatter(), gapPolicy, metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java

@@ -108,7 +108,7 @@ public GapPolicy gapPolicy() {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new BucketSelectorPipelineAggregator(name, bucketsPathsMap, script, gapPolicy, metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketSortPipelineAggregationBuilder.java

@@ -139,7 +139,7 @@ public BucketSortPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new BucketSortPipelineAggregator(name, sorts, from, size, gapPolicy, metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumPipelineAggregationBuilder.java

@@ -88,7 +88,7 @@ protected DocValueFormat formatter() {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new CumulativeSumPipelineAggregator(name, bucketsPaths, formatter(), metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/DerivativePipelineAggregationBuilder.java

@@ -129,7 +129,7 @@ public String unit() {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         DocValueFormat formatter;
         if (format != null) {
             formatter = new DocValueFormat.Decimal(format);

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.java

@@ -75,7 +75,7 @@ public double sigma() {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new ExtendedStatsBucketPipelineAggregator(name, bucketsPaths, sigma, gapPolicy(), formatter(), metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MaxBucketPipelineAggregationBuilder.java

@@ -46,7 +46,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new MaxBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MinBucketPipelineAggregationBuilder.java

@@ -46,7 +46,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new MinBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java

@@ -250,7 +250,7 @@ public Boolean minimize() {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         // If the user doesn't set a preference for cost minimization, ask
         // what the model prefers
         boolean minimize = this.minimize == null ? model.minimizeByDefault() : this.minimize;

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovFnPipelineAggregationBuilder.java

@@ -179,7 +179,7 @@ public void doValidate(AggregatorFactory<?> parent, Collection<AggregationBuilde
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new MovFnPipelineAggregator(name, bucketsPathString, script, window, formatter(), gapPolicy, metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/PercentilesBucketPipelineAggregationBuilder.java

@@ -112,7 +112,7 @@ public boolean getKeyed() {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new PercentilesBucketPipelineAggregator(name, percents, keyed, bucketsPaths, gapPolicy(), formatter(), metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SerialDiffPipelineAggregationBuilder.java

@@ -135,7 +135,7 @@ protected DocValueFormat formatter() {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new SerialDiffPipelineAggregator(name, bucketsPaths, formatter(), gapPolicy, lag, metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/StatsBucketPipelineAggregationBuilder.java

@@ -47,7 +47,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new StatsBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData);
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SumBucketPipelineAggregationBuilder.java

@@ -46,7 +46,7 @@ protected void innerWriteTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    protected PipelineAggregator createInternal(Map<String, Object> metaData) throws IOException {
+    protected PipelineAggregator createInternal(Map<String, Object> metaData) {
         return new SumBucketPipelineAggregator(name, bucketsPaths, gapPolicy(), formatter(), metaData);
     }