Merge branch 'master' into ccr

* master: (68 commits)
  [DOCS] Removes X-Pack Elasticsearch release notes (#30272)
  Correct an example in the top-level suggester documentation. (#30224)
  [DOCS] Removes broken link
  [DOCS] Adds file realm configuration details (#30221)
  [DOCS] Adds PKI realm configuration details (#30225)
  Fix a reference to match_phrase_prefix in the match query docs. (#30282)
  Fix failure for validate API on a terms query (#29483)
  [DOCS] Fix 6.4-specific link in changelog (#30314)
  Remove RepositoriesMetaData variadic constructor (#29569)
  Test: increase authentication logging for debugging
  [DOCS] Removes redundant SAML realm settings (#30196)
  REST Client: Add Request object flavored methods (#29623)
  [DOCS] Adds changelog to Elasticsearch Reference (#30271)
  [DOCS] Fixes section error
  SQL: Teach the CLI to ignore empty commands (#30265)
  [DOCS] Adds Active Directory realm configuration details (#30223)
  [DOCS] Removes redundant file realm settings (#30192)
  [DOCS] Fixes users command name (#30275)
  Build: Move gradle wrapper jar to a dot dir (#30146)
  Build: Log a warning if disabling reindex-from-old (#30304)

Author: dnhatn

gradle/wrapper/gradle-wrapper.jar renamed to .gradle-wrapper/gradle-wrapper.jar

@@ -1,6 +1,6 @@
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-all.zip
 zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
 distributionSha256Sum=203f4537da8b8075e38c036a6d14cb71b1149de5bf0a8f6db32ac2833a1d1294

gradle/wrapper/gradle-wrapper.properties renamed to .gradle-wrapper/gradle-wrapper.properties

@@ -337,6 +337,7 @@ export BATS=/project/build/bats
 export BATS_UTILS=/project/build/packaging/bats/utils
 export BATS_TESTS=/project/build/packaging/bats/tests
 export PACKAGING_ARCHIVES=/project/build/packaging/archives
+export PACKAGING_TESTS=/project/build/packaging/tests
 VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
 Defaults   env_keep += "ZIP"
@@ -347,6 +348,7 @@ Defaults   env_keep += "BATS"
 Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
 Defaults   env_keep += "PACKAGING_ARCHIVES"
+Defaults   env_keep += "PACKAGING_TESTS"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL

Vagrantfile

@@ -440,25 +440,18 @@ task run(type: Run) {
   impliesSubProjects = true
 }
 
-task wrapper(type: Wrapper)
-
-gradle.projectsEvaluated {
-
-  allprojects {
-    tasks.withType(Wrapper) { Wrapper wrapper ->
-      wrapper.distributionType = DistributionType.ALL
-
-      wrapper.doLast {
-        final DistributionLocator locator = new DistributionLocator()
-        final GradleVersion version = GradleVersion.version(wrapper.gradleVersion)
-        final URI distributionUri = locator.getDistributionFor(version, wrapper.distributionType.name().toLowerCase(Locale.ENGLISH))
-        final URI sha256Uri = new URI(distributionUri.toString() + ".sha256")
-        final String sha256Sum = new String(sha256Uri.toURL().bytes)
-        wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n"
-      }
-    }
+task wrapper(type: Wrapper) {
+  distributionType = DistributionType.ALL
+  jarFile = file('.gradle-wrapper/gradle-wrapper.jar')
+  doLast {
+    final DistributionLocator locator = new DistributionLocator()
+    final GradleVersion version = GradleVersion.version(gradleVersion)
+    final URI distributionUri = locator.getDistributionFor(version, distributionType.name().toLowerCase(Locale.ENGLISH))
+    final URI sha256Uri = new URI(distributionUri.toString() + ".sha256")
+    final String sha256Sum = new String(sha256Uri.toURL().bytes)
+    final String existingProperties = getPropertiesFile().getText('UTF-8')
+    getPropertiesFile().setText("${existingProperties}distributionSha256Sum=${sha256Sum}\n", 'UTF-8')
   }
-
 }
 
 static void assertLinesInFile(final Path path, final List<String> expectedLines) {

build.gradle

@@ -549,6 +549,22 @@ class BuildPlugin implements Plugin<Project> {
             javadoc.classpath = javadoc.getClasspath().filter { f ->
                 return classes.contains(f) == false
             }
+            /*
+             * Force html5 on projects that support it to silence the warning
+             * that `javadoc` will change its defaults in the future.
+             *
+             * But not all of our javadoc is actually valid html5. So we
+             * have to become valid incrementally. We only set html5 on the
+             * projects we have converted so that we still get the annoying
+             * warning on the unconverted ones. That will give us an
+             * incentive to convert them....
+             */
+            List html4Projects = [
+                ':server',
+            ]
+            if (false == html4Projects.contains(project.path)) {
+                javadoc.options.addBooleanOption('html5', true)
+            }
         }
         configureJavadocJar(project)
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -41,6 +41,9 @@ class VagrantPropertiesExtension {
     @Input
     Boolean inheritTestUtils
 
+    @Input
+    String testClass
+
     VagrantPropertiesExtension(List<String> availableBoxes) {
         this.boxes = availableBoxes
         this.batsDir = 'src/test/resources/packaging'

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy

@@ -51,6 +51,7 @@ class VagrantTestPlugin implements Plugin<Project> {
     static List<String> UPGRADE_FROM_ARCHIVES = ['rpm', 'deb']
 
     private static final PACKAGING_CONFIGURATION = 'packaging'
+    private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest'
     private static final BATS = 'bats'
     private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
     private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && ./gradlew test integTest"
@@ -66,6 +67,7 @@ class VagrantTestPlugin implements Plugin<Project> {
 
         // Creates custom configurations for Bats testing files (and associated scripts and archives)
         createPackagingConfiguration(project)
+        project.configurations.create(PACKAGING_TEST_CONFIGURATION)
 
         // Creates all the main Vagrant tasks
         createVagrantTasks(project)
@@ -144,10 +146,12 @@ class VagrantTestPlugin implements Plugin<Project> {
     }
 
     private static void createCleanTask(Project project) {
-        project.tasks.create('clean', Delete.class) {
-            description 'Clean the project build directory'
-            group 'Build'
-            delete project.buildDir
+        if (project.tasks.findByName('clean') == null) {
+            project.tasks.create('clean', Delete.class) {
+                description 'Clean the project build directory'
+                group 'Build'
+                delete project.buildDir
+            }
         }
     }
 
@@ -174,6 +178,18 @@ class VagrantTestPlugin implements Plugin<Project> {
             from project.configurations[PACKAGING_CONFIGURATION]
         }
 
+        File testsDir = new File(packagingDir, 'tests')
+        Copy copyPackagingTests = project.tasks.create('copyPackagingTests', Copy) {
+            into testsDir
+            from project.configurations[PACKAGING_TEST_CONFIGURATION]
+        }
+
+        Task createTestRunnerScript = project.tasks.create('createTestRunnerScript', FileContentsTask) {
+            dependsOn copyPackagingTests
+            file "${testsDir}/run-tests.sh"
+            contents "java -cp \"\$PACKAGING_TESTS/*\" org.junit.runner.JUnitCore ${-> project.extensions.esvagrant.testClass}"
+        }
+
         Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
             dependsOn copyPackagingArchives
             file "${archivesDir}/version"
@@ -234,7 +250,8 @@ class VagrantTestPlugin implements Plugin<Project> {
 
         Task vagrantSetUpTask = project.tasks.create('setupPackagingTest')
         vagrantSetUpTask.dependsOn 'vagrantCheckVersion'
-        vagrantSetUpTask.dependsOn copyPackagingArchives, createVersionFile, createUpgradeFromFile, createUpgradeIsOssFile
+        vagrantSetUpTask.dependsOn copyPackagingArchives, copyPackagingTests, createTestRunnerScript
+        vagrantSetUpTask.dependsOn createVersionFile, createUpgradeFromFile, createUpgradeIsOssFile
         vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils
     }
 
@@ -393,20 +410,29 @@ class VagrantTestPlugin implements Plugin<Project> {
                 packagingTest.dependsOn(batsPackagingTest)
             }
 
-            // This task doesn't do anything yet. In the future it will execute a jar containing tests on the vm
-            Task groovyPackagingTest = project.tasks.create("vagrant${boxTask}#groovyPackagingTest")
-            groovyPackagingTest.dependsOn(up)
-            groovyPackagingTest.finalizedBy(halt)
+            Task javaPackagingTest = project.tasks.create("vagrant${boxTask}#javaPackagingTest", VagrantCommandTask) {
+                command 'ssh'
+                boxName box
+                environmentVars vagrantEnvVars
+                dependsOn up, setupPackagingTest
+                finalizedBy halt
+                args '--command', "bash \"\$PACKAGING_TESTS/run-tests.sh\""
+            }
+
+            // todo remove this onlyIf after all packaging tests are consolidated
+            javaPackagingTest.onlyIf {
+                project.extensions.esvagrant.testClass != null
+            }
 
-            TaskExecutionAdapter groovyPackagingReproListener = createReproListener(project, groovyPackagingTest.path)
-            groovyPackagingTest.doFirst {
-                project.gradle.addListener(groovyPackagingReproListener)
+            TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
+            javaPackagingTest.doFirst {
+                project.gradle.addListener(javaPackagingReproListener)
             }
-            groovyPackagingTest.doLast {
-                project.gradle.removeListener(groovyPackagingReproListener)
+            javaPackagingTest.doLast {
+                project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                packagingTest.dependsOn(groovyPackagingTest)
+                packagingTest.dependsOn(javaPackagingTest)
             }
 
             Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -129,15 +129,17 @@ public NoopBulkRequestBuilder setWaitForActiveShards(final int waitForActiveShar
     }
 
     /**
-     * A timeout to wait if the index operation can't be performed immediately. Defaults to <tt>1m</tt>.
+     * A timeout to wait if the index operation can't be performed immediately.
+     * Defaults to {@code 1m}.
      */
     public final NoopBulkRequestBuilder setTimeout(TimeValue timeout) {
         request.timeout(timeout);
         return this;
     }
 
     /**
-     * A timeout to wait if the index operation can't be performed immediately. Defaults to <tt>1m</tt>.
+     * A timeout to wait if the index operation can't be performed immediately.
+     * Defaults to {@code 1m}.
      */
     public final NoopBulkRequestBuilder setTimeout(String timeout) {
         request.timeout(timeout);
@@ -151,4 +153,3 @@ public int numberOfActions() {
         return request.numberOfActions();
     }
 }
-

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java

@@ -142,7 +142,7 @@ public NoopSearchRequestBuilder setRouting(String... routing) {
 
     /**
      * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards or a custom value, which guarantees that the same order
+     * {@code _local} to prefer local shards or a custom value, which guarantees that the same order
      * will be used across different requests.
      */
     public NoopSearchRequestBuilder setPreference(String preference) {
@@ -188,15 +188,15 @@ public NoopSearchRequestBuilder setMinScore(float minScore) {
     }
 
     /**
-     * From index to start the search from. Defaults to <tt>0</tt>.
+     * From index to start the search from. Defaults to {@code 0}.
      */
     public NoopSearchRequestBuilder setFrom(int from) {
         sourceBuilder().from(from);
         return this;
     }
 
     /**
-     * The number of search hits to return. Defaults to <tt>10</tt>.
+     * The number of search hits to return. Defaults to {@code 10}.
      */
     public NoopSearchRequestBuilder setSize(int size) {
         sourceBuilder().size(size);
@@ -349,7 +349,7 @@ public NoopSearchRequestBuilder slice(SliceBuilder builder) {
 
     /**
      * Applies when sorting, and controls if scores will be tracked as well. Defaults to
-     * <tt>false</tt>.
+     * {@code false}.
      */
     public NoopSearchRequestBuilder setTrackScores(boolean trackScores) {
         sourceBuilder().trackScores(trackScores);

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchRequestBuilder.java

@@ -48,7 +48,7 @@ public final class ClusterClient {
      */
     public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, Header... headers)
             throws IOException {
-        return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, Request::clusterPutSettings,
+        return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings,
                 ClusterUpdateSettingsResponse::fromXContent, emptySet(), headers);
     }
 
@@ -60,7 +60,7 @@ public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest cl
      */
     public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest,
             ActionListener<ClusterUpdateSettingsResponse> listener, Header... headers) {
-        restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, Request::clusterPutSettings,
+        restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings,
                 ClusterUpdateSettingsResponse::fromXContent, listener, emptySet(), headers);
     }
 }