Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (170 commits)
  Allow TrimFilter to be used in custom normalizers (#27758)
  recovery from snapshot should fill gaps (#27850)
  Remove unused class PreBuiltTokenFilters (#27839)
  Reject scroll query if size is 0 (#22552) (#27842)
  Mutes ‘Rollover no condition matched’ YAML test
  Make randomNonNegativeLong() draw from a uniform distribution (#27856)
  Adapt rest test after backport. Relates #27833
  Handle case where the hole vertex is south of the containing polygon(s) (#27685)
  Move range field mapper back to core
  Fix publication of elasticsearch-cli to Maven
  Do not use system properties when building the HttpAsyncClient (#27829)
  Optimize version map for append-only indexing (#27752)
  Add NioGroup for use in different transports (#27737)
  adapt field collapsing skip test version. relates #27833
  Add version support for inner hits in field collapsing (#27822) (#27833)
  Clarify that number of threads is set by packages
  Register HTTP read timeout setting
  Fixes Checkstyle
  Remove `operationThreaded` from Java API (#27836)
  Fixes failing BytesSizeValues tests
  ...

Author: martijnvg

CONTRIBUTING.md

@@ -106,11 +106,19 @@ IntelliJ users can automatically configure their IDE: `gradle idea`
 then `File->New Project From Existing Sources`. Point to the root of
 the source directory, select
 `Import project from external model->Gradle`, enable
-`Use auto-import`. Additionally, in order to run tests directly from 
-IDEA 2017.1 and above it is required to disable IDEA run launcher, 
-which can be achieved by adding `-Didea.no.launcher=true` 
-[JVM option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) 
-
+`Use auto-import`. In order to run tests directly from
+IDEA 2017.2 and above, it is required to disable the IDEA run launcher in order to avoid
+`idea_rt.jar` causing "jar hell". This can be achieved by adding the
+`-Didea.no.launcher=true` [JVM
+option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties).
+Alternatively, `idea.no.launcher=true` can be set in the
+[`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
+file which can be accessed under Help > Edit Custom Properties (this will require a
+restart of IDEA). For IDEA 2017.3 and above, in addition to the JVM option, you will need to go to
+`Run->Edit Configurations...` and change the value for the `Shorten command line` setting from
+`user-local default: none` to `classpath file`. You may also need to [remove `ant-javafx.jar` from your
+classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
+reported as a source of jar hell.
 
 The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This

Vagrantfile

@@ -307,4 +307,9 @@ Defaults   env_keep += "BATS_ARCHIVES"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL
+  # This prevents leftovers from previous tests using the
+  # same VM from messing up the current test
+  config.vm.provision "clean_tmp", run: "always", type: "shell", inline: <<-SHELL
+    rm -rf /tmp/elasticsearch*
+  SHELL
 end

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -131,12 +131,6 @@ class BuildPlugin implements Plugin<Project> {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }
 
-            final GradleVersion gradle42 = GradleVersion.version('4.2')
-            final GradleVersion gradle43 = GradleVersion.version('4.3')
-            if (currentGradleVersion >= gradle42 && currentGradleVersion < gradle43) {
-                throw new GradleException("${currentGradleVersion} is not compatible with the elasticsearch build")
-            }
-
             // enforce Java version
             if (javaVersionEnum < minimumJava) {
                 throw new GradleException("Java ${minimumJava} or above is required to build Elasticsearch")

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionCollection.groovy

@@ -29,6 +29,7 @@ import java.util.regex.Matcher
 class VersionCollection {
 
     private final List<Version> versions
+    private final boolean buildSnapshot = System.getProperty("build.snapshot", "true") == "true"
 
     /**
      * Construct a VersionCollection from the lines of the Version.java file.
@@ -63,7 +64,10 @@ class VersionCollection {
             throw new GradleException("Unexpectedly found no version constants in Versions.java");
         }
 
-        // The tip of each minor series (>= 5.6) is unreleased, so set their 'snapshot' flags
+        /*
+         * The tip of each minor series (>= 5.6) is unreleased, so they must be built from source (we set branch to non-null), and we set
+         * the snapshot flag if and only if build.snapshot is true.
+         */
         Version prevConsideredVersion = null
         boolean found6xSnapshot = false
         for (final int versionIndex = versions.size() - 1; versionIndex >= 0; versionIndex--) {
@@ -85,7 +89,7 @@ class VersionCollection {
 
                 versions[versionIndex] = new Version(
                         currConsideredVersion.major, currConsideredVersion.minor,
-                        currConsideredVersion.revision, currConsideredVersion.suffix, true, branch)
+                        currConsideredVersion.revision, currConsideredVersion.suffix, buildSnapshot, branch)
             }
 
             if (currConsideredVersion.onOrBefore("5.6.0")) {
@@ -95,12 +99,6 @@ class VersionCollection {
             prevConsideredVersion = currConsideredVersion
         }
 
-        // If we're making a release build then the current should not be a snapshot after all.
-        final boolean currentIsSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        if (false == currentIsSnapshot) {
-            versions[-1] = new Version(versions[-1].major, versions[-1].minor, versions[-1].revision, versions[-1].suffix, false, null)
-        }
-
         this.versions = Collections.unmodifiableList(versions)
     }
 
@@ -137,7 +135,7 @@ class VersionCollection {
     private Version getLastSnapshotWithMajor(int targetMajor) {
         final String currentVersion = currentVersion.toString()
         final int snapshotIndex = versions.findLastIndexOf {
-            it.major == targetMajor && it.before(currentVersion) && it.snapshot
+            it.major == targetMajor && it.before(currentVersion) && it.snapshot == buildSnapshot
         }
         return snapshotIndex == -1 ? null : versions[snapshotIndex]
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy

@@ -69,7 +69,6 @@ public class AntFixture extends AntTask implements Fixture {
      * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
      * condition is for http on the http port.
      */
-    @Input
     Closure waitCondition = { AntFixture fixture, AntBuilder ant ->
         File tmpFile = new File(fixture.cwd, 'wait.success')
         ant.get(src: "http://${fixture.addressAndPort}",

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -166,11 +166,13 @@ class ClusterFormationTasks {
         Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
             delete node.homeDir
             delete node.cwd
+        }
+        setup = project.tasks.create(name: taskName(prefix, node, 'createCwd'), type: DefaultTask, dependsOn: setup) {
             doLast {
                 node.cwd.mkdirs()
             }
+            outputs.dir node.cwd
         }
-
         setup = configureCheckPreviousTask(taskName(prefix, node, 'checkPrevious'), project, setup, node)
         setup = configureStopTask(taskName(prefix, node, 'stopPrevious'), project, setup, node)
         setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
@@ -267,33 +269,6 @@ class ClusterFormationTasks {
                     into node.baseDir
                 }
                 break;
-            case 'rpm':
-                File rpmDatabase = new File(node.baseDir, 'rpm-database')
-                File rpmExtracted = new File(node.baseDir, 'rpm-extracted')
-                /* Delay reading the location of the rpm file until task execution */
-                Object rpm = "${ -> configuration.singleFile}"
-                extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
-                    commandLine 'rpm', '--badreloc', '--nodeps', '--noscripts', '--notriggers',
-                        '--dbpath', rpmDatabase,
-                        '--relocate', "/=${rpmExtracted}",
-                        '-i', rpm
-                    doFirst {
-                        rpmDatabase.deleteDir()
-                        rpmExtracted.deleteDir()
-                    }
-                }
-                break;
-            case 'deb':
-                /* Delay reading the location of the deb file until task execution */
-                File debExtracted = new File(node.baseDir, 'deb-extracted')
-                Object deb = "${ -> configuration.singleFile}"
-                extract = project.tasks.create(name: name, type: LoggedExec, dependsOn: extractDependsOn) {
-                    commandLine 'dpkg-deb', '-x', deb, debExtracted
-                    doFirst {
-                        debExtracted.deleteDir()
-                    }
-                }
-                break;
             default:
                 throw new InvalidUserDataException("Unknown distribution: ${node.config.distribution}")
         }

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -658,7 +658,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]RelocationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]TruncatedRecoveryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]BytesRestResponseTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]AliasResolveRoutingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]AliasRoutingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]SimpleRoutingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]FileScriptTests.java" checks="LineLength" />

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 6.2.0
-lucene            = 7.1.0
+lucene            = 7.2.0-snapshot-7deca62
 
 # optional dependencies
 spatial4j         = 0.6

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -21,21 +21,25 @@
 
 import org.apache.http.Header;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 
 import java.io.IOException;
 import java.util.Collections;
 
 /**
  * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Indices API.
- *
+ * <p>
  * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html">Indices API on elastic.co</a>
  */
 public final class IndicesClient {
     private final RestHighLevelClient restHighLevelClient;
 
-    public IndicesClient(RestHighLevelClient restHighLevelClient) {
+    IndicesClient(RestHighLevelClient restHighLevelClient) {
         this.restHighLevelClient = restHighLevelClient;
     }
 
@@ -56,8 +60,55 @@ public DeleteIndexResponse deleteIndex(DeleteIndexRequest deleteIndexRequest, He
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-delete-index.html">
      * Delete Index API on elastic.co</a>
      */
-    public void deleteIndexAsync(DeleteIndexRequest deleteIndexRequest, ActionListener<DeleteIndexResponse> listener, Header... headers) {
+    public void deleteIndexAsync(DeleteIndexRequest deleteIndexRequest, ActionListener<DeleteIndexResponse> listener,
+                                       Header... headers) {
         restHighLevelClient.performRequestAsyncAndParseEntity(deleteIndexRequest, Request::deleteIndex, DeleteIndexResponse::fromXContent,
             listener, Collections.emptySet(), headers);
     }
+
+    /**
+     * Creates an index using the Create Index API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html">
+     * Create Index API on elastic.co</a>
+     */
+    public CreateIndexResponse createIndex(CreateIndexRequest createIndexRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(createIndexRequest, Request::createIndex, CreateIndexResponse::fromXContent,
+                Collections.emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously creates an index using the Create Index API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html">
+     * Create Index API on elastic.co</a>
+     */
+    public void createIndexAsync(CreateIndexRequest createIndexRequest, ActionListener<CreateIndexResponse> listener,
+                                       Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(createIndexRequest, Request::createIndex, CreateIndexResponse::fromXContent,
+            listener, Collections.emptySet(), headers);
+    }
+
+    /**
+     * Opens an index using the Open Index API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-open-close.html">
+     * Open Index API on elastic.co</a>
+     */
+    public OpenIndexResponse openIndex(OpenIndexRequest openIndexRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(openIndexRequest, Request::openIndex, OpenIndexResponse::fromXContent,
+                Collections.emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously opens an index using the Open Index API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-open-close.html">
+     * Open Index API on elastic.co</a>
+     */
+    public void openIndexAsync(OpenIndexRequest openIndexRequest, ActionListener<OpenIndexResponse> listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(openIndexRequest, Request::openIndex, OpenIndexResponse::fromXContent,
+                listener, Collections.emptySet(), headers);
+    }
+
 }