Merge branch 'master' into resync-less-lenient

Author: dnhatn

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy

@@ -100,12 +100,15 @@ class TestProgressLogger implements AggregatedEventListener {
 
     @Subscribe
     void onQuit(AggregatedQuitEvent e) throws IOException {
-        suiteLogger.completed()
-        testLogger.completed()
-        for (ProgressLogger slaveLogger : slaveLoggers) {
-            slaveLogger.completed()
+        // if onStart was never called (eg no matching tests), suiteLogger and all the other loggers will be null
+        if (suiteLogger != null) {
+            suiteLogger.completed()
+            testLogger.completed()
+            for (ProgressLogger slaveLogger : slaveLoggers) {
+                slaveLogger.completed()
+            }
+            parentProgressLogger.completed()
         }
-        parentProgressLogger.completed()
     }
 
     @Subscribe

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -430,7 +430,7 @@ class BuildPlugin implements Plugin<Project> {
                 // we fork because compiling lots of different classes in a shared jvm can eventually trigger GC overhead limitations
                 options.fork = true
                 options.forkOptions.javaHome = new File(project.compilerJavaHome)
-                options.forkOptions.memoryMaximumSize = "1g"
+                options.forkOptions.memoryMaximumSize = "512m"
                 if (targetCompatibilityVersion == JavaVersion.VERSION_1_8) {
                     // compile with compact 3 profile by default
                     // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -106,6 +106,7 @@ public class PluginBuildPlugin extends BuildPlugin {
     private static void createIntegTestTask(Project project) {
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
         integTest.mustRunAfter(project.precommit, project.test)
+        project.integTestCluster.distribution = 'integ-test-zip'
         project.check.dependsOn(integTest)
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -28,7 +28,7 @@ class ClusterConfiguration {
     private final Project project
 
     @Input
-    String distribution = 'integ-test-zip'
+    String distribution = 'zip'
 
     @Input
     int numNodes = 1

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy

@@ -46,7 +46,6 @@ public class RestTestPlugin implements Plugin<Project> {
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
         integTest.description = 'Runs rest tests against an elasticsearch cluster.'
         integTest.group = JavaBasePlugin.VERIFICATION_GROUP
-        integTest.clusterConfig.distribution = 'zip' // rest tests should run with the real zip
         integTest.mustRunAfter(project.precommit)
         project.check.dependsOn(integTest)
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -3,6 +3,7 @@ package org.elasticsearch.gradle.vagrant
 import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.FileContentsTask
+import org.elasticsearch.gradle.LoggedExec
 import org.gradle.api.*
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.execution.TaskExecutionAdapter
@@ -309,15 +310,35 @@ class VagrantTestPlugin implements Plugin<Project> {
             }
             update.mustRunAfter(setupBats)
 
+            /*
+             * Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag
+             * vagrant.destroy that defaults to true that can be used to control whether or not to destroy any test boxes before test
+             * execution.
+             */
+            final String vagrantDestroyProperty = project.getProperties().get('vagrant.destroy', 'true')
+            final boolean vagrantDestroy
+            if ("true".equals(vagrantDestroyProperty)) {
+                vagrantDestroy = true;
+            } else if ("false".equals(vagrantDestroyProperty)) {
+                vagrantDestroy = false
+            } else {
+                throw new GradleException("[vagrant.destroy] must be [true] or [false] but was [" + vagrantDestroyProperty + "]")
+            }
+            /*
+             * Some versions of Vagrant will fail destroy if the box does not exist. Therefore we check if the box exists before attempting
+             * to destroy the box.
+             */
+            final Task destroy = project.tasks.create("vagrant${boxTask}#destroy", LoggedExec) {
+                commandLine "bash", "-c", "vagrant status ${box} | grep -q \"${box}\\s\\+not created\" || vagrant destroy ${box} --force"
+                workingDir project.rootProject.rootDir
+            }
+            destroy.onlyIf { vagrantDestroy }
+            update.mustRunAfter(destroy)
+
             Task up = project.tasks.create("vagrant${boxTask}#up", VagrantCommandTask) {
                 command 'up'
                 boxName box
                 environmentVars vagrantEnvVars
-                /* Its important that we try to reprovision the box even if it already
-                  exists. That way updates to the vagrant configuration take automatically.
-                  That isn't to say that the updates will always be compatible. Its ok to
-                  just destroy the boxes if they get busted but that is a manual step
-                  because its slow-ish. */
                 /* We lock the provider to virtualbox because the Vagrantfile specifies
                   lots of boxes that only work properly in virtualbox. Virtualbox is
                   vagrant's default but its possible to change that default and folks do.
@@ -326,7 +347,7 @@ class VagrantTestPlugin implements Plugin<Project> {
                 args '--provision', '--provider', 'virtualbox'
                 /* It'd be possible to check if the box is already up here and output
                   SKIPPED but that would require running vagrant status which is slow! */
-                dependsOn update
+                dependsOn destroy, update
             }
 
             Task smoke = project.tasks.create("vagrant${boxTask}#smoketest", Exec) {

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -10,9 +10,9 @@
 #
 # classname=foo.bar.BazPlugin
 # description=My cool plugin
-# version=2.0
-# elasticsearch.version=2.0
-# java.version=1.7
+# version=6.0
+# elasticsearch.version=6.0
+# java.version=1.8
 #
 ### mandatory elements for all plugins:
 #

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -30,13 +30,17 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeResponse;
 
@@ -215,6 +219,46 @@ public void existsAliasAsync(GetAliasesRequest getAliasesRequest, ActionListener
                 listener, emptySet(), headers);
     }
 
+    /**
+     * Refresh one or more indices using the Refresh API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html"> Refresh API on elastic.co</a>
+     */
+    public RefreshResponse refresh(RefreshRequest refreshRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(refreshRequest, Request::refresh, RefreshResponse::fromXContent,
+                emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously refresh one or more indices using the Refresh API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html"> Refresh API on elastic.co</a>
+     */
+    public void refreshAsync(RefreshRequest refreshRequest, ActionListener<RefreshResponse> listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(refreshRequest, Request::refresh, RefreshResponse::fromXContent,
+                listener, emptySet(), headers);
+    }
+
+    /**
+     * Flush one or more indices using the Flush API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-flush.html"> Flush API on elastic.co</a>
+     */
+    public FlushResponse flush(FlushRequest flushRequest, Header... headers) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(flushRequest, Request::flush, FlushResponse::fromXContent,
+                emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously flush one or more indices using the Flush API
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-flush.html"> Flush API on elastic.co</a>
+     */
+    public void flushAsync(FlushRequest flushRequest, ActionListener<FlushResponse> listener, Header... headers) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(flushRequest, Request::flush, FlushResponse::fromXContent,
+                listener, emptySet(), headers);
+    }
+
     /**
      * Checks if the index (indices) exists or not.
      * <p>

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -35,9 +35,11 @@
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
@@ -216,6 +218,22 @@ static Request putMapping(PutMappingRequest putMappingRequest) throws IOExceptio
         return new Request(HttpPut.METHOD_NAME, endpoint, parameters.getParams(), entity);
     }
 
+    static Request refresh(RefreshRequest refreshRequest) {
+        String endpoint = endpoint(refreshRequest.indices(), "_refresh");
+        Params parameters = Params.builder();
+        parameters.withIndicesOptions(refreshRequest.indicesOptions());
+        return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null);
+    }
+
+    static Request flush(FlushRequest flushRequest) {
+        String endpoint = endpoint(flushRequest.indices(), "_flush");
+        Params parameters = Params.builder();
+        parameters.withIndicesOptions(flushRequest.indicesOptions());
+        parameters.putParam("wait_if_ongoing", Boolean.toString(flushRequest.waitIfOngoing()));
+        parameters.putParam("force", Boolean.toString(flushRequest.force()));
+        return new Request(HttpPost.METHOD_NAME, endpoint, parameters.getParams(), null);
+    }
+
     static Request info() {
         return new Request(HttpGet.METHOD_NAME, "/", Collections.emptyMap(), null);
     }

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -34,11 +34,15 @@
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
@@ -47,6 +51,7 @@
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -381,6 +386,58 @@ public void testCloseNonExistentIndex() throws IOException {
         assertEquals(RestStatus.NOT_FOUND, exception.status());
     }
 
+    public void testRefresh() throws IOException {
+        {
+            String index = "index";
+            Settings settings = Settings.builder()
+                .put("number_of_shards", 1)
+                .put("number_of_replicas", 0)
+                .build();
+            createIndex(index, settings);
+            RefreshRequest refreshRequest = new RefreshRequest(index);
+            RefreshResponse refreshResponse =
+                execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync);
+            assertThat(refreshResponse.getTotalShards(), equalTo(1));
+            assertThat(refreshResponse.getSuccessfulShards(), equalTo(1));
+            assertThat(refreshResponse.getFailedShards(), equalTo(0));
+            assertThat(refreshResponse.getShardFailures(), equalTo(BroadcastResponse.EMPTY));
+        }
+        {
+            String nonExistentIndex = "non_existent_index";
+            assertFalse(indexExists(nonExistentIndex));
+            RefreshRequest refreshRequest = new RefreshRequest(nonExistentIndex);
+            ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync));
+            assertEquals(RestStatus.NOT_FOUND, exception.status());
+        }
+    }
+
+    public void testFlush() throws IOException {
+        {
+            String index = "index";
+            Settings settings = Settings.builder()
+                    .put("number_of_shards", 1)
+                    .put("number_of_replicas", 0)
+                    .build();
+            createIndex(index, settings);
+            FlushRequest flushRequest = new FlushRequest(index);
+            FlushResponse flushResponse =
+                    execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync);
+            assertThat(flushResponse.getTotalShards(), equalTo(1));
+            assertThat(flushResponse.getSuccessfulShards(), equalTo(1));
+            assertThat(flushResponse.getFailedShards(), equalTo(0));
+            assertThat(flushResponse.getShardFailures(), equalTo(BroadcastResponse.EMPTY));
+        }
+        {
+            String nonExistentIndex = "non_existent_index";
+            assertFalse(indexExists(nonExistentIndex));
+            FlushRequest refreshRequest = new FlushRequest(nonExistentIndex);
+            ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                    () -> execute(refreshRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync));
+            assertEquals(RestStatus.NOT_FOUND, exception.status());
+        }
+    }
+
     public void testExistsAlias() throws IOException {
         GetAliasesRequest getAliasesRequest = new GetAliasesRequest("alias");
         assertFalse(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync));
@@ -495,4 +552,4 @@ public void testRollover() throws IOException {
             assertEquals("test_new", rolloverResponse.getNewIndex());
         }
     }
-}
+}