Merge remote-tracking branch 'origin/master' into snapshot-lifecycle-management

Author: dakrone

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -89,6 +89,7 @@ class PluginBuildPlugin extends BuildPlugin {
                 project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName ->
                     // Auto add dependent modules to the test cluster
                     if (project.findProject(":modules:${pluginName}") != null) {
+                        project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin)
                         project.testClusters.integTest.module(
                                 project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile)
                         )

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -936,8 +936,6 @@ class ClusterFormationTasks {
             }
             doLast {
                 project.delete(node.pidFile)
-                // Large tests can exhaust disk space, clean up on stop, but leave the data dir as some tests reuse it
-                project.delete(project.fileTree(node.baseDir).minus(project.fileTree(node.dataDir)))
             }
         }
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -120,7 +120,6 @@ class RestIntegTestTask extends DefaultTask {
             if (usesTestclusters == true) {
                 ElasticsearchCluster cluster = project.testClusters."${name}"
                 nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }")
-                nonInputProperties.systemProperty('tests.config.dir', "${-> cluster.singleNode().getConfigDir() }")
                 nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }")
             } else {
                 // we pass all nodes to the rest cluster to allow the clients to round-robin between them

buildSrc/src/main/java/org/elasticsearch/gradle/FileSupplier.java

@@ -0,0 +1,7 @@
+package org.elasticsearch.gradle;
+
+import java.io.File;
+import java.util.function.Supplier;
+
+public interface FileSupplier extends Supplier<File> {
+}

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -20,6 +20,7 @@
 
 import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;
+import org.elasticsearch.gradle.FileSupplier;
 import org.elasticsearch.gradle.Version;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Project;
@@ -143,6 +144,16 @@ public void keystore(String key, Supplier<CharSequence> valueSupplier) {
         nodes.all(each -> each.keystore(key, valueSupplier));
     }
 
+    @Override
+    public void keystore(String key, File value) {
+        nodes.all(each -> each.keystore(key, value));
+    }
+
+    @Override
+    public void keystore(String key, FileSupplier valueSupplier) {
+        nodes.all(each -> each.keystore(key, valueSupplier));
+    }
+
     @Override
     public void setting(String key, String value) {
         nodes.all(each -> each.setting(key, value));

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -20,6 +20,7 @@
 
 import org.elasticsearch.GradleServicesAdapter;
 import org.elasticsearch.gradle.Distribution;
+import org.elasticsearch.gradle.FileSupplier;
 import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.Version;
 import org.gradle.api.logging.Logger;
@@ -63,7 +64,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final int NODE_UP_TIMEOUT = 60;
     private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.SECONDS;
     private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList(
-        "path.repo"
+        "path.repo",
+        "discovery.seed_providers"
     );
 
     private final String path;
@@ -79,6 +81,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final List<File> modules = new ArrayList<>();
     private final Map<String, Supplier<CharSequence>> settings = new LinkedHashMap<>();
     private final Map<String, Supplier<CharSequence>> keystoreSettings = new LinkedHashMap<>();
+    private final Map<String, FileSupplier> keystoreFiles = new LinkedHashMap<>();
     private final Map<String, Supplier<CharSequence>> systemProperties = new LinkedHashMap<>();
     private final Map<String, Supplier<CharSequence>> environment = new LinkedHashMap<>();
     private final Map<String, File> extraConfigFiles = new HashMap<>();
@@ -171,6 +174,19 @@ public void keystore(String key, Supplier<CharSequence> valueSupplier) {
         addSupplier("Keystore", keystoreSettings, key, valueSupplier);
     }
 
+    @Override
+    public void keystore(String key, File value) {
+        requireNonNull(value, "keystore value was null when configuring test cluster`" + this + "`");
+        keystore(key, () -> value);
+    }
+
+    @Override
+    public void keystore(String key, FileSupplier valueSupplier) {
+        requireNonNull(key, "Keystore" + " key was null when configuring test cluster `" + this + "`");
+        requireNonNull(valueSupplier, "Keystore" + " value supplier was null when configuring test cluster `" + this + "`");
+        keystoreFiles.put(key, valueSupplier);
+    }
+
     @Override
     public void setting(String key, String value) {
         addSupplier("Settings", settings, key, value);
@@ -281,12 +297,22 @@ public synchronized void start() {
             "install", "--batch", plugin.toString())
         );
 
-        if (keystoreSettings.isEmpty() == false) {
-            checkSuppliers("Keystore", keystoreSettings);
+        if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             runElaticsearchBinScript("elasticsearch-keystore", "create");
-            keystoreSettings.forEach((key, value) -> {
-                runElaticsearchBinScriptWithInput(value.get().toString(), "elasticsearch-keystore", "add", "-x", key);
-            });
+
+            checkSuppliers("Keystore", keystoreSettings);
+            keystoreSettings.forEach((key, value) ->
+                runElaticsearchBinScriptWithInput(value.get().toString(), "elasticsearch-keystore", "add", "-x", key)
+            );
+
+            for (Map.Entry<String, FileSupplier> entry : keystoreFiles.entrySet()) {
+                File file = entry.getValue().get();
+                requireNonNull(file, "supplied keystoreFile was null when configuring " + this);
+                if (file.exists() == false) {
+                    throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
+                }
+                runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
+            }
         }
 
         installModules();

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle.testclusters;
 
 import org.elasticsearch.gradle.Distribution;
+import org.elasticsearch.gradle.FileSupplier;
 import org.gradle.api.logging.Logging;
 import org.slf4j.Logger;
 
@@ -47,6 +48,10 @@ public interface TestClusterConfiguration {
 
     void keystore(String key, Supplier<CharSequence> valueSupplier);
 
+    void keystore(String key, File value);
+
+    void keystore(String key, FileSupplier valueSupplier);
+
     void setting(String key, String value);
 
     void setting(String key, Supplier<CharSequence> valueSupplier);

buildSrc/version.properties

@@ -37,7 +37,7 @@ httpcore          = 4.4.11
 httpasyncclient   = 4.1.4
 commonslogging    = 1.1.3
 commonscodec      = 1.11
-hamcrest          = 1.3
+hamcrest          = 2.1
 securemock        = 1.2
 mocksocket        = 1.2
 

client/rest-high-level/build.gradle

@@ -61,7 +61,6 @@ dependencies {
   testCompile "org.elasticsearch.test:framework:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
-  testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
   //this is needed to make RestHighLevelClientTests#testApiNamingConventions work from IDEs
   testCompile "org.elasticsearch:rest-api-spec:${version}"
   // Needed for serialization tests: 

client/rest-high-level/src/main/java/org/elasticsearch/client/DataFrameClient.java

@@ -49,7 +49,8 @@ public final class DataFrameClient {
      * Creates a new Data Frame Transform
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Data Frame PUT transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/put-data-frame-transform.html">
+     *     Create data frame transform documentation</a>
      *
      * @param request The PutDataFrameTransformRequest containing the
      * {@link org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig}.
@@ -69,7 +70,8 @@ public AcknowledgedResponse putDataFrameTransform(PutDataFrameTransformRequest r
      * Creates a new Data Frame Transform asynchronously and notifies listener on completion
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Data Frame PUT transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/put-data-frame-transform.html">
+     *     Create data frame transform documentation</a>
      *
      * @param request The PutDataFrameTransformRequest containing the
      * {@link org.elasticsearch.client.dataframe.transforms.DataFrameTransformConfig}.
@@ -90,7 +92,8 @@ public void putDataFrameTransformAsync(PutDataFrameTransformRequest request, Req
      * Get the running statistics of a Data Frame Transform
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Get Data Frame transform stats documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/get-data-frame-transform-stats.html">
+     *     Get data frame transform stats documentation</a>
      *
      * @param request Specifies the which transforms to get the stats for
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -110,7 +113,8 @@ public GetDataFrameTransformStatsResponse getDataFrameTransformStats(GetDataFram
      * Get the running statistics of a Data Frame Transform asynchronously and notifies listener on completion
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Get Data Frame transform stats documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/get-data-frame-transform-stats.html">
+     *     Get data frame transform stats documentation</a>
      *
      * @param request Specifies the which transforms to get the stats for
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -130,7 +134,8 @@ public void getDataFrameTransformStatsAsync(GetDataFrameTransformStatsRequest re
      * Delete a data frame transform
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Data Frame delete transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/delete-data-frame-transform.html">
+     *     Delete data frame transform documentation</a>
      *
      * @param request The delete data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -150,7 +155,8 @@ public AcknowledgedResponse deleteDataFrameTransform(DeleteDataFrameTransformReq
      * Delete a data frame transform asynchronously and notifies listener on completion
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Data Frame delete transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/delete-data-frame-transform.html">
+     *     Delete data frame transform documentation</a>
      *
      * @param request The delete data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -170,7 +176,8 @@ public void deleteDataFrameTransformAsync(DeleteDataFrameTransformRequest reques
      * Preview the result of a data frame transform
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Preview Data Frame transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/preview-data-frame-transform.html">
+     *     Preview data frame transform documentation</a>
      *
      * @param request The preview data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -189,8 +196,8 @@ public PreviewDataFrameTransformResponse previewDataFrameTransform(PreviewDataFr
     /**
      * Preview the result of a data frame transform asynchronously and notifies listener on completion
      * <p>
-     * For additional info
-     * see <a href="https://www.TODO.com">Preview Data Frame transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/preview-data-frame-transform.html">
+     *     Preview data frame transform documentation</a>
      *
      * @param request The preview data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -210,7 +217,8 @@ public void previewDataFrameTransformAsync(PreviewDataFrameTransformRequest requ
      * Start a data frame transform
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Start Data Frame transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/start-data-frame-transform.html">
+     *     Start data frame transform documentation</a>
      *
      * @param request The start data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -230,7 +238,8 @@ public StartDataFrameTransformResponse startDataFrameTransform(StartDataFrameTra
      * Start a data frame transform asynchronously and notifies listener on completion
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Start Data Frame transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/start-data-frame-transform.html">
+     *     Start data frame transform documentation</a>
      *
      * @param request The start data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -250,7 +259,8 @@ public void startDataFrameTransformAsync(StartDataFrameTransformRequest request,
      * Stop a data frame transform
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Stop Data Frame transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/stop-data-frame-transform.html">
+     *     Stop data frame transform documentation</a>
      *
      * @param request The stop data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -270,7 +280,8 @@ public StopDataFrameTransformResponse stopDataFrameTransform(StopDataFrameTransf
      * Stop a data frame transform asynchronously and notifies listener on completion
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Stop Data Frame transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/stop-data-frame-transform.html">
+     *     Stop data frame transform documentation</a>
      *
      * @param request The stop data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -290,7 +301,8 @@ public void stopDataFrameTransformAsync(StopDataFrameTransformRequest request, R
      * Get one or more data frame transform configurations
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Get Data Frame transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/get-data-frame-transform.html">
+     *     Get data frame transform documentation</a>
      *
      * @param request The get data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
@@ -310,7 +322,8 @@ public GetDataFrameTransformResponse getDataFrameTransform(GetDataFrameTransform
      * Get one or more data frame transform configurations asynchronously and notifies listener on completion
      * <p>
      * For additional info
-     * see <a href="https://www.TODO.com">Get Data Frame transform documentation</a>
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/get-data-frame-transform.html">
+     *     Get data frame transform documentation</a>
      *
      * @param request The get data frame transform request
      * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized