Merge branch 'master' into 2018-12-11-no-master-block-integration-tests

Author: DaveCTurner

buildSrc/build.gradle

@@ -95,6 +95,7 @@ dependencies {
   }
   minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
   minimumRuntimeCompile localGroovy()
+  minimumRuntimeCompile gradleApi()
 }
 jar {
   from sourceSets.minimumRuntime.output

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -57,6 +57,7 @@ import org.gradle.util.GradleVersion
 import java.nio.charset.StandardCharsets
 import java.time.ZoneOffset
 import java.time.ZonedDateTime
+import java.util.regex.Matcher
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -265,56 +266,68 @@ class BuildPlugin implements Plugin<Project> {
             rootProject.rootProject.ext.buildDocker = buildDocker
             rootProject.rootProject.ext.requiresDocker = []
             rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
-                int exitCode
-                String dockerErrorOutput
-                if (dockerBinary == null) {
-                    exitCode = -1
-                    dockerErrorOutput = null
-                } else {
-                    // the Docker binary executes, check that we can execute a privileged command
-                    final ByteArrayOutputStream output = new ByteArrayOutputStream()
-                    final ExecResult result = LoggedExec.exec(rootProject, { ExecSpec it ->
-                        it.commandLine dockerBinary, "images"
-                        it.errorOutput = output
-                        it.ignoreExitValue = true
-                    })
-                    if (result.exitValue == 0) {
-                        return
-                    }
-                    exitCode = result.exitValue
-                    dockerErrorOutput = output.toString()
-                }
                 final List<String> tasks =
                         ((List<Task>)rootProject.requiresDocker).findAll { taskGraph.hasTask(it) }.collect { "  ${it.path}".toString()}
                 if (tasks.isEmpty() == false) {
                     /*
                      * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
                      * exist or because execution of a privileged Docker command failed.
                      */
-                    String message
                     if (dockerBinary == null) {
-                        message = String.format(
+                        final String message = String.format(
                                 Locale.ROOT,
                                 "Docker (checked [%s]) is required to run the following task%s: \n%s",
                                 maybeDockerBinaries.join(","),
                                 tasks.size() > 1 ? "s" : "",
                                 tasks.join('\n'))
-                    } else {
-                        assert exitCode > 0 && dockerErrorOutput != null
-                        message = String.format(
+                        throwDockerRequiredException(message)
+                    }
+
+                    // we use a multi-stage Docker build, check the Docker version since 17.05
+                    final ByteArrayOutputStream dockerVersionOutput = new ByteArrayOutputStream()
+                    LoggedExec.exec(
+                            rootProject,
+                            { ExecSpec it ->
+                                it.commandLine = [dockerBinary, '--version']
+                                it.standardOutput = dockerVersionOutput
+                            })
+                    final String dockerVersion = dockerVersionOutput.toString().trim()
+                    final Matcher matcher = dockerVersion =~ /Docker version (\d+\.\d+)\.\d+(?:-ce)?, build [0-9a-f]{7}/
+                    assert matcher.matches() : dockerVersion
+                    final dockerMajorMinorVersion = matcher.group(1)
+                    final String[] majorMinor = dockerMajorMinorVersion.split("\\.")
+                    if (Integer.parseInt(majorMinor[0]) < 17
+                            || (Integer.parseInt(majorMinor[0]) == 17 && Integer.parseInt(majorMinor[1]) < 5)) {
+                        final String message = String.format(
+                                Locale.ROOT,
+                                "building Docker images requires Docker version 17.05+ due to use of multi-stage builds yet was [%s]",
+                                dockerVersion)
+                        throwDockerRequiredException(message)
+                    }
+
+                    final ByteArrayOutputStream dockerImagesErrorOutput = new ByteArrayOutputStream()
+                    // the Docker binary executes, check that we can execute a privileged command
+                    final ExecResult dockerImagesResult = LoggedExec.exec(
+                            rootProject,
+                            { ExecSpec it ->
+                                it.commandLine = [dockerBinary, "images"]
+                                it.errorOutput = dockerImagesErrorOutput
+                                it.ignoreExitValue = true
+                            })
+
+                    if (dockerImagesResult.exitValue != 0) {
+                        final String message = String.format(
                                 Locale.ROOT,
                                 "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
                                         "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
                                 dockerBinary,
                                 tasks.size() > 1 ? "s" : "",
                                 tasks.join('\n'),
-                                exitCode,
-                                dockerErrorOutput.trim())
+                                dockerImagesResult.exitValue,
+                                dockerImagesErrorOutput.toString().trim())
+                        throwDockerRequiredException(message)
                     }
-                    throw new GradleException(
-                            message + "\nyou can address this by attending to the reported issue, "
-                                    + "removing the offending tasks from being executed, "
-                                    + "or by passing -Dbuild.docker=false")
+
                 }
             }
         }
@@ -325,6 +338,13 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
+    private static void throwDockerRequiredException(final String message) {
+        throw new GradleException(
+                message + "\nyou can address this by attending to the reported issue, "
+                        + "removing the offending tasks from being executed, "
+                        + "or by passing -Dbuild.docker=false")
+    }
+
     private static String findCompilerJavaHome() {
         String compilerJavaHome = System.getenv('JAVA_HOME')
         final String compilerJavaProperty = System.getProperty('compiler.java')

buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java renamed to buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/LoggedExec.java

@@ -75,14 +75,22 @@ protected static RestHighLevelClient highLevelClient() {
      */
     protected static <Req, Resp> Resp execute(Req request, SyncMethod<Req, Resp> syncMethod,
                                        AsyncMethod<Req, Resp> asyncMethod) throws IOException {
+        return execute(request, syncMethod, asyncMethod, RequestOptions.DEFAULT);
+    }
+    
+    /**
+     * Executes the provided request using either the sync method or its async variant, both provided as functions
+     */
+    protected static <Req, Resp> Resp execute(Req request, SyncMethod<Req, Resp> syncMethod,
+                                       AsyncMethod<Req, Resp> asyncMethod, RequestOptions options) throws IOException {
         if (randomBoolean()) {
-            return syncMethod.execute(request, RequestOptions.DEFAULT);
+            return syncMethod.execute(request, options);
         } else {
             PlainActionFuture<Resp> future = PlainActionFuture.newFuture();
-            asyncMethod.execute(request, RequestOptions.DEFAULT, future);
+            asyncMethod.execute(request, options, future);
             return future.actionGet();
         }
-    }
+    }    
 
     /**
      * Executes the provided request using either the sync method or its async

client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java

@@ -1254,6 +1254,14 @@ public void testCountAllIndicesNoQuery() throws IOException {
         CountRequest countRequest = new CountRequest();
         CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync);
         assertCountHeader(countResponse);
+        // add logging to get more info about why https://github.com/elastic/elasticsearch/issues/35644 is failing
+        // TODO remove this once #35644 is fixed
+        if (countResponse.getCount() != 12) {
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.source(new SearchSourceBuilder().size(20));
+            SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            logger.info("Unexpected hit count, was expecting 12 hits but got: " + searchResponse.toString());
+        }
         assertEquals(12, countResponse.getCount());
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -88,7 +88,7 @@ This API doesn't allow you to move the documents it touches, just modify their
 source. This is intentional! We've made no provisions for removing the document
 from its original location.
 
-You can also perform these operations on multiple indices and types at once, similar to the search API:
+You can also perform these operations on multiple indices at once, similar to the search API:
 
 ["source","java",subs="attributes,callouts,macros"]
 --------------------------------------------------

docs/java-api/docs/update-by-query.asciidoc

@@ -6,8 +6,10 @@
 
 This section summarizes the changes in each release.
 
+* <<release-notes-7.0.0-alpha2>>
 * <<release-notes-7.0.0-alpha1>>
 
 --
 
+include::release-notes/7.0.0-alpha2.asciidoc[]
 include::release-notes/7.0.0-alpha1.asciidoc[]

docs/reference/release-notes.asciidoc

@@ -1,5 +1,5 @@
 [[release-notes-7.0.0-alpha1]]
-== 7.0.0-alpha1 release notes
+== {es} version 7.0.0-alpha1
 
 The changes listed below have been released for the first time in Elasticsearch 7.0.0-alpha1.
 

docs/reference/release-notes/7.0.0-alpha1.asciidoc

@@ -0,0 +1,4 @@
+[[release-notes-7.0.0-alpha2]]
+== {es} version 7.0.0-alpha2
+
+coming[7.0.0-alpha2]

docs/reference/release-notes/7.0.0-alpha2.asciidoc

@@ -41,21 +41,37 @@ public static PainlessCast getLegalCast(Location location, Class<?> actual, Clas
 
         if (actual == def.class) {
             if (expected == boolean.class) {
-                return PainlessCast.unboxTargetType(def.class, Boolean.class, explicit, boolean.class);
+                return PainlessCast.originalTypetoTargetType(def.class, boolean.class, explicit);
             } else if (expected == byte.class) {
-                return PainlessCast.unboxTargetType(def.class, Byte.class, explicit, byte.class);
+                return PainlessCast.originalTypetoTargetType(def.class, byte.class, explicit);
             } else if (expected == short.class) {
-                return PainlessCast.unboxTargetType(def.class, Short.class, explicit, short.class);
+                return PainlessCast.originalTypetoTargetType(def.class, short.class, explicit);
             } else if (expected == char.class) {
-                return PainlessCast.unboxTargetType(def.class, Character.class, explicit, char.class);
+                return PainlessCast.originalTypetoTargetType(def.class, char.class, explicit);
             } else if (expected == int.class) {
-                return PainlessCast.unboxTargetType(def.class, Integer.class, explicit, int.class);
+                return PainlessCast.originalTypetoTargetType(def.class, int.class, explicit);
             } else if (expected == long.class) {
-                return PainlessCast.unboxTargetType(def.class, Long.class, explicit, long.class);
+                return PainlessCast.originalTypetoTargetType(def.class, long.class, explicit);
             } else if (expected == float.class) {
-                return PainlessCast.unboxTargetType(def.class, Float.class, explicit, float.class);
+                return PainlessCast.originalTypetoTargetType(def.class, float.class, explicit);
             } else if (expected == double.class) {
-                return PainlessCast.unboxTargetType(def.class, Double.class, explicit, double.class);
+                return PainlessCast.originalTypetoTargetType(def.class, double.class, explicit);
+            } else if (expected == Boolean.class) {
+                return PainlessCast.originalTypetoTargetType(def.class, Boolean.class, explicit);
+            } else if (expected == Byte.class) {
+                return PainlessCast.originalTypetoTargetType(def.class, Byte.class, explicit);
+            } else if (expected == Short.class) {
+                return PainlessCast.originalTypetoTargetType(def.class, Short.class, explicit);
+            } else if (expected == Character.class) {
+                return PainlessCast.originalTypetoTargetType(def.class, Character.class, explicit);
+            } else if (expected == Integer.class) {
+                return PainlessCast.originalTypetoTargetType(def.class, Integer.class, explicit);
+            } else if (expected == Long.class) {
+                return PainlessCast.originalTypetoTargetType(def.class, Long.class, explicit);
+            } else if (expected == Float.class) {
+                return PainlessCast.originalTypetoTargetType(def.class, Float.class, explicit);
+            } else if (expected == Double.class) {
+                return PainlessCast.originalTypetoTargetType(def.class, Double.class, explicit);
             }
         } else if (actual == Object.class) {
             if (expected == byte.class && explicit && internal) {