Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (89 commits)
  Clarify requirements of strict date formats. (#29090)
  Clarify that dates are always rendered as strings. (#29093)
  [Docs] Fix link to Grok patterns (#29088)
  Fix starting on Windows from another drive (#29086)
  Use removeTask instead of finishTask in PersistentTasksClusterService (#29055)
  Added minimal docs for reindex api in java-api docs
  Allow overriding JVM options in Windows service (#29044)
  Clarify how to set compiler and runtime JDKs (#29101)
  Fix Parsing Bug with Update By Query for Stored Scripts (#29039)
  TEST: write ops should execute under shard permit (#28966)
  [DOCS] Add X-Pack upgrade details (#29038)
  Revert "Improve error message for installing plugin (#28298)"
  Docs: HighLevelRestClient#exists (#29073)
  Validate regular expressions in dynamic templates. (#29013)
  [Tests] Fix GetResultTests and DocumentFieldTests failures (#29083)
  Reenable LiveVersionMapTests.testRamBytesUsed on Java 9. (#29063)
  Mute failing GetResultTests and DocumentFieldTests
  [Docs] Fix Java Api index administration usage (#28260)
  Improve error message for installing plugin (#28298)
  Decouple XContentBuilder from BytesReference (#28972)
  ...

Author: martijnvg

TESTING.asciidoc

@@ -270,14 +270,14 @@ The REST tests are run automatically when executing the "./gradlew check" comman
 REST tests use the following command:
 
 ---------------------------------------------------------------------------
-./gradlew :distribution:integ-test-zip:integTest   \
+./gradlew :distribution:archives:integ-test-zip:integTest   \
   -Dtests.class="org.elasticsearch.test.rest.*Yaml*IT"
 ---------------------------------------------------------------------------
 
 A specific test case can be run with
 
 ---------------------------------------------------------------------------
-./gradlew :distribution:integ-test-zip:integTest \
+./gradlew :distribution:archives:integ-test-zip:integTest \
   -Dtests.class="org.elasticsearch.test.rest.*Yaml*IT" \
   -Dtests.method="test {p0=cat.shards/10_basic/Help}"
 ---------------------------------------------------------------------------
@@ -407,7 +407,7 @@ destructive. When working with a single package it's generally faster to run its
 tests in a tighter loop than Gradle provides. In one window:
 
 --------------------------------
-./gradlew :distribution:rpm:assemble
+./gradlew :distribution:packages:rpm:assemble
 --------------------------------
 
 and in another window:

benchmarks/build.gradle

@@ -85,6 +85,10 @@ shadowJar {
     classifier = 'benchmarks'
 }
 
+runShadow {
+    executable = new File(project.runtimeJavaHome, 'bin/java')
+}
+
 // alias the shadowJar and runShadow tasks to abstract from the concrete plugin that we are using and provide a more consistent interface
 task jmhJar(
         dependsOn: shadowJar,

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -140,16 +140,22 @@ class BuildPlugin implements Plugin<Project> {
 
             final GradleVersion minGradle = GradleVersion.version('4.3')
             if (currentGradleVersion < minGradle) {
-                throw new GradleException("${minGradle} or above is required to build elasticsearch")
+                throw new GradleException("${minGradle} or above is required to build Elasticsearch")
             }
 
             // enforce Java version
             if (compilerJavaVersionEnum < minimumCompilerVersion) {
-                throw new GradleException("Java ${minimumCompilerVersion} or above is required to build Elasticsearch")
+                final String message =
+                        "the environment variable JAVA_HOME must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
+                                " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"
+                throw new GradleException(message)
             }
 
             if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
-                throw new GradleException("Java ${minimumRuntimeVersion} or above is required to run Elasticsearch")
+                final String message =
+                        "the environment variable RUNTIME_JAVA_HOME must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
+                                " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"
+                throw new GradleException(message)
             }
 
             project.rootProject.ext.compilerJavaHome = compilerJavaHome
@@ -558,7 +564,7 @@ class BuildPlugin implements Plugin<Project> {
         return {
             jvm "${project.runtimeJavaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', 'auto')
-            ifNoTests 'fail'
+            ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
 
@@ -582,8 +588,6 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
-            // default test sysprop values
-            systemProperty 'tests.ifNoTests', 'fail'
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {

buildSrc/src/main/resources/forbidden/es-all-signatures.txt

@@ -48,3 +48,5 @@ java.nio.channels.SocketChannel#connect(java.net.SocketAddress)
 # org.elasticsearch.common.Booleans#parseBoolean(java.lang.String) directly on the string.
 @defaultMessage use org.elasticsearch.common.Booleans#parseBoolean(java.lang.String)
 java.lang.Boolean#getBoolean(java.lang.String)
+
+org.apache.lucene.util.IOUtils @ use @org.elasticsearch.core.internal.io instead

buildSrc/src/main/resources/forbidden/es-server-signatures.txt

@@ -29,8 +29,6 @@ java.util.concurrent.Executors#privilegedThreadFactory()
 java.lang.Character#codePointBefore(char[],int) @ Implicit start offset is error-prone when the char[] is a buffer and the first chars are random chars
 java.lang.Character#codePointAt(char[],int) @ Implicit end offset is error-prone when the char[] is a buffer and the last chars are random chars
 
-java.io.StringReader#<init>(java.lang.String) @ Use FastStringReader instead
-
 @defaultMessage Reference management is tricky, leave it to SearcherManager
 org.apache.lucene.index.IndexReader#decRef()
 org.apache.lucene.index.IndexReader#incRef()
@@ -135,3 +133,17 @@ java.time.OffsetDateTime#withYear(int)
 java.time.zone.ZoneRules#getStandardOffset(java.time.Instant)
 java.time.zone.ZoneRules#getDaylightSavings(java.time.Instant)
 java.time.zone.ZoneRules#isDaylightSavings(java.time.Instant)
+
+@defaultMessage Use logger methods with non-Object parameter
+org.apache.logging.log4j.Logger#trace(java.lang.Object)
+org.apache.logging.log4j.Logger#trace(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#debug(java.lang.Object)
+org.apache.logging.log4j.Logger#debug(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#info(java.lang.Object)
+org.apache.logging.log4j.Logger#info(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#warn(java.lang.Object)
+org.apache.logging.log4j.Logger#warn(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#error(java.lang.Object)
+org.apache.logging.log4j.Logger#error(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#fatal(java.lang.Object)
+org.apache.logging.log4j.Logger#fatal(java.lang.Object, java.lang.Throwable)

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -43,6 +43,3 @@ extended.plugins=${extendedPlugins}
 #
 # 'has.native.controller': whether or not the plugin has a native controller
 has.native.controller=${hasNativeController}
-#
-# 'requires.keystore': whether or not the plugin needs the elasticsearch keystore be created
-requires.keystore=${requiresKeystore}

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -330,7 +330,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                 }
                 metadata.endObject();
 
-                BytesRef metadataSource = metadata.bytes().toBytesRef();
+                BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
                 content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
                 content.write(separator);
             }
@@ -345,7 +345,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                     LoggingDeprecationHandler.INSTANCE, indexSource, indexXContentType)) {
                     try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                         builder.copyCurrentStructure(parser);
-                        source = builder.bytes().toBytesRef();
+                        source = BytesReference.bytes(builder).toBytesRef();
                     }
                 }
             } else if (opType == DocWriteRequest.OpType.UPDATE) {

client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

@@ -617,7 +617,8 @@ public void testBulk() throws IOException {
                 bulkRequest.add(deleteRequest);
 
             } else {
-                BytesReference source = XContentBuilder.builder(xContentType.xContent()).startObject().field("id", i).endObject().bytes();
+                BytesReference source = BytesReference.bytes(XContentBuilder.builder(xContentType.xContent())
+                        .startObject().field("id", i).endObject());
                 if (opType == DocWriteRequest.OpType.INDEX) {
                     IndexRequest indexRequest = new IndexRequest("index", "test", id).source(source, xContentType);
                     if (erroneous) {

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -55,6 +55,7 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -272,7 +273,7 @@ private static HttpEntity createBinaryEntity(XContentBuilder xContentBuilder, Co
             builder.startObject();
             builder.field("field", "value");
             builder.endObject();
-            return new ByteArrayEntity(builder.bytes().toBytesRef().bytes, contentType);
+            return new ByteArrayEntity(BytesReference.bytes(builder).toBytesRef().bytes, contentType);
         }
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
@@ -478,7 +479,7 @@ public void testSearchScroll() throws Exception {
 
         for (int i = 0; i < 100; i++) {
             XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
-            HttpEntity entity = new NStringEntity(builder.string(), ContentType.APPLICATION_JSON);
+            HttpEntity entity = new NStringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
             client().performRequest(HttpPut.METHOD_NAME, "test/type1/" + Integer.toString(i), Collections.emptyMap(), entity);
         }
         client().performRequest(HttpPost.METHOD_NAME, "/test/_refresh");