Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (43 commits)
  ingest: upgraded ingest geoip's geoip2's dependencies.
  [TEST] logging for update by query test #27820
  Use full profile on JDK 10 builds
  Require Gradle 4.3
  Add unreleased v6.1.2 version
  TEST: reduce blob size #testExecuteMultipartUpload
  Check index under the store metadata lock (#27768)
  Upgrade to Lucene 7.2.0. (#27910)
  Fixed test to be up to date with the new database files.
  Use `_refresh` to shrink the version map on inactivity (#27918)
  Make KeyedLock reentrant (#27920)
  Fixes DocStats to not report index size < -1 (#27863)
  Disable TestZenDiscovery in cloud providers integrations test
  ingest: Upgraded the geolite2 databases.
  [Issue-27716]: CONTRIBUTING.md IntelliJ configurations settings are confusing. (#27717)
  [Test] Fix IndicesClientDocumentationIT (#27899)
  Move uid lock into LiveVersionMap (#27905)
  Mute testRetentionPolicyChangeDuringRecovery
  Increase Gradle heap space to 1536m
  Move GlobalCheckpointTracker and remove SequenceNumbersService (#27837)
  ...

Author: martijnvg

CONTRIBUTING.md

@@ -115,11 +115,13 @@ Alternatively, `idea.no.launcher=true` can be set in the
 [`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
 file which can be accessed under Help > Edit Custom Properties (this will require a
 restart of IDEA). For IDEA 2017.3 and above, in addition to the JVM option, you will need to go to
-`Run->Edit Configurations...` and change the value for the `Shorten command line` setting from
+`Run->Edit Configurations->...->Defaults->JUnit` and change the value for the `Shorten command line` setting from
 `user-local default: none` to `classpath file`. You may also need to [remove `ant-javafx.jar` from your
 classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
 reported as a source of jar hell.
 
+To run an instance of elasticsearch from the source code run `gradle run`
+
 The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This
 can be accomplished by passing the flag `-ea` to the JVM on startup.

build.gradle

@@ -196,6 +196,7 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
+    "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
 
   for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -126,7 +126,7 @@ class BuildPlugin implements Plugin<Project> {
             // enforce Gradle version
             final GradleVersion currentGradleVersion = GradleVersion.current();
 
-            final GradleVersion minGradle = GradleVersion.version('3.3')
+            final GradleVersion minGradle = GradleVersion.version('4.3')
             if (currentGradleVersion < minGradle) {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }
@@ -408,7 +408,11 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Adds compiler settings to the project */
     static void configureCompile(Project project) {
-        project.ext.compactProfile = 'compact3'
+        if (project.javaVersion < JavaVersion.VERSION_1_10) {
+            project.ext.compactProfile = 'compact3'
+        } else {
+            project.ext.compactProfile = 'full'
+        }
         project.afterEvaluate {
             project.tasks.withType(JavaCompile) {
                 File gradleJavaHome = Jvm.current().javaHome
@@ -444,13 +448,6 @@ class BuildPlugin implements Plugin<Project> {
                     // hack until gradle supports java 9's new "--release" arg
                     assert minimumJava == JavaVersion.VERSION_1_8
                     options.compilerArgs << '--release' << '8'
-                    if (GradleVersion.current().getBaseVersion() < GradleVersion.version("4.1")) {
-                        // this hack is not needed anymore since Gradle 4.1, see https://github.com/gradle/gradle/pull/2474
-                        doFirst {
-                            sourceCompatibility = null
-                            targetCompatibility = null
-                        }
-                    }
                 }
             }
         }

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 6.2.0
-lucene            = 7.2.0-snapshot-7deca62
+lucene            = 7.2.0
 
 # optional dependencies
 spatial4j         = 0.6

client/rest-high-level/build.gradle

@@ -39,6 +39,7 @@ dependencies {
   compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
   compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
+  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
 
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
@@ -60,4 +61,4 @@ forbiddenApisMain {
   // specified
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
   signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
-}
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -57,6 +57,10 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.cbor.CborXContent;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
+import org.elasticsearch.index.rankeval.DiscountedCumulativeGain;
+import org.elasticsearch.index.rankeval.EvaluationMetric;
+import org.elasticsearch.index.rankeval.MeanReciprocalRank;
+import org.elasticsearch.index.rankeval.PrecisionAtK;
 import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHits;
@@ -648,7 +652,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(2, namedXContents.size());
+        assertEquals(5, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -658,10 +662,14 @@ public void testProvidedNamedXContents() {
                 categories.put(namedXContent.categoryClass, counter + 1);
             }
         }
-        assertEquals(1, categories.size());
+        assertEquals(2, categories.size());
         assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
+        assertEquals(Integer.valueOf(3), categories.get(EvaluationMetric.class));
+        assertTrue(names.contains(PrecisionAtK.NAME));
+        assertTrue(names.contains(DiscountedCumulativeGain.NAME));
+        assertTrue(names.contains(MeanReciprocalRank.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener<Integer> {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -86,6 +87,32 @@ public void testDeleteIndex() throws IOException {
             boolean acknowledged = deleteIndexResponse.isAcknowledged(); // <1>
             // end::delete-index-response
             assertTrue(acknowledged);
+        }
+
+        {
+            // tag::delete-index-notfound
+            try {
+                DeleteIndexRequest request = new DeleteIndexRequest("does_not_exist");
+                client.indices().deleteIndex(request);
+            } catch (ElasticsearchException exception) {
+                if (exception.status() == RestStatus.NOT_FOUND) {
+                    // <1>
+                }
+            }
+            // end::delete-index-notfound
+        }
+    }
+
+    public void testDeleteIndexAsync() throws Exception {
+        final RestHighLevelClient client = highLevelClient();
+
+        {
+            CreateIndexResponse createIndexResponse = client.indices().createIndex(new CreateIndexRequest("posts"));
+            assertTrue(createIndexResponse.isAcknowledged());
+        }
+
+        {
+            DeleteIndexRequest request = new DeleteIndexRequest("posts");
 
             // tag::delete-index-execute-async
             client.indices().deleteIndexAsync(request, new ActionListener<DeleteIndexResponse>() {
@@ -100,19 +127,12 @@ public void onFailure(Exception e) {
                 }
             });
             // end::delete-index-execute-async
-        }
 
-        {
-            // tag::delete-index-notfound
-            try {
-                DeleteIndexRequest request = new DeleteIndexRequest("does_not_exist");
-                client.indices().deleteIndex(request);
-            } catch (ElasticsearchException exception) {
-                if (exception.status() == RestStatus.NOT_FOUND) {
-                    // <1>
-                }
-            }
-            // end::delete-index-notfound
+            assertBusy(() -> {
+                // TODO Use Indices Exist API instead once it exists
+                Response response = client.getLowLevelClient().performRequest("HEAD", "posts");
+                assertTrue(RestStatus.NOT_FOUND.getStatus() == response.getStatusLine().getStatusCode());
+            });
         }
     }
 
@@ -174,6 +194,14 @@ public void testCreateIndex() throws IOException {
             // end::create-index-response
             assertTrue(acknowledged);
             assertTrue(shardsAcked);
+        }
+    }
+
+    public void testCreateIndexAsync() throws Exception {
+        final RestHighLevelClient client = highLevelClient();
+
+        {
+            CreateIndexRequest request = new CreateIndexRequest("twitter");
 
             // tag::create-index-execute-async
             client.indices().createIndexAsync(request, new ActionListener<CreateIndexResponse>() {
@@ -188,6 +216,13 @@ public void onFailure(Exception e) {
                 }
             });
             // end::create-index-execute-async
+
+            assertBusy(() -> {
+                // TODO Use Indices Exist API instead once it exists
+                Response response = client.getLowLevelClient().performRequest("HEAD", "twitter");
+                assertTrue(RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode());
+            });
         }
     }
+
 }

client/transport/build.gradle

@@ -32,6 +32,7 @@ dependencies {
   compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
   compile "org.elasticsearch.plugin:percolator-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
+  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
@@ -54,4 +55,4 @@ namingConventions {
   testClass = 'com.carrotsearch.randomizedtesting.RandomizedTest'
   //we don't have integration tests
   skipIntegTestInDisguise = true
-}
+}

core/licenses/lucene-analyzers-common-7.2.0-snapshot-7deca62.jar.sha1

@@ -0,0 +1 @@
+4e1b4638fb8b07befc8175880641f821af3e655a