Merge remote-tracking branch 'es/7.x' into enrich-7.x

Author: martijnvg

build.gradle

@@ -33,7 +33,7 @@ import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
 
 plugins {
     id 'com.gradle.build-scan' version '2.4'
-    id 'base'
+    id 'lifecycle-base'
     id 'elasticsearch.global-build-info'
 }
 
@@ -179,8 +179,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/45767"
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.gradle
 
-
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import groovy.transform.CompileDynamic
@@ -48,7 +47,6 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
-import org.gradle.api.artifacts.repositories.ArtifactRepository
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository
 import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
 import org.gradle.api.artifacts.repositories.MavenArtifactRepository
@@ -408,11 +406,11 @@ class BuildPlugin implements Plugin<Project> {
         project.getRepositories().all { repository ->
             if (repository instanceof MavenArtifactRepository) {
                 final MavenArtifactRepository maven = (MavenArtifactRepository) repository
-                assertRepositoryURIUsesHttps(maven, project, maven.getUrl())
-                repository.getArtifactUrls().each { uri -> assertRepositoryURIUsesHttps(maven, project, uri) }
+                assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl())
+                repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) }
             } else if (repository instanceof IvyArtifactRepository) {
                 final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
-                assertRepositoryURIUsesHttps(ivy, project, ivy.getUrl())
+                assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl())
             }
         }
         RepositoryHandler repos = project.repositories
@@ -452,9 +450,15 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
-    private static void assertRepositoryURIUsesHttps(final ArtifactRepository repository, final Project project, final URI uri) {
-        if (uri != null && uri.toURL().getProtocol().equals("http")) {
-            throw new GradleException("repository [${repository.name}] on project with path [${project.path}] is using http for artifacts on [${uri.toURL()}]")
+    static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
+        if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) {
+            final String message = String.format(
+                    Locale.ROOT,
+                    "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
+                    repositoryName,
+                    projectPath,
+                    uri.toURL())
+            throw new GradleException(message)
         }
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -123,7 +123,7 @@ public class SnippetsTask extends DefaultTask {
                     }
                 }
                 if (snippet.testResponse
-                        && 'js' == snippet.language
+                        && ('js' == snippet.language || 'console-result' == snippet.language)
                         && null == snippet.skip) {
                     String quoted = snippet.contents
                         // quote values starting with $
@@ -162,7 +162,7 @@ public class SnippetsTask extends DefaultTask {
                     }
                     return
                 }
-                matcher = line =~ /\["?source"?,\s*"?(\w+)"?(,.*)?].*/
+                matcher = line =~ /\["?source"?,\s*"?([-\w]+)"?(,.*)?].*/
                 if (matcher.matches()) {
                     lastLanguage = matcher.group(1)
                     lastLanguageLine = lineNumber

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -387,7 +387,7 @@ class ClusterFormationTasks {
         Map esConfig = [
                 'cluster.name'                 : node.clusterName,
                 'node.name'                    : "node-" + node.nodeNum,
-                'pidfile'                      : node.pidFile,
+                (node.nodeVersion.onOrAfter('7.4.0') ? 'node.pidfile' : 'pidfile') : node.pidFile,
                 'path.repo'                    : "${node.sharedDir}/repo",
                 'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -105,6 +105,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         "is a pre-release version of Elasticsearch",
         "max virtual memory areas vm.max_map_count"
     );
+    private static final String HOSTNAME_OVERRIDE = "LinuxDarwinHostname";
+    private static final String COMPUTERNAME_OVERRIDE = "WindowsComputername";
 
     private final String path;
     private final String name;
@@ -604,6 +606,10 @@ private Map<String, String> getESEnvironment() {
         // Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR
         defaultEnv.put("TMP", tmpDir.toString());
 
+        // Override the system hostname variables for testing
+        defaultEnv.put("HOSTNAME", HOSTNAME_OVERRIDE);
+        defaultEnv.put("COMPUTERNAME", COMPUTERNAME_OVERRIDE);
+
         Set<String> commonKeys = new HashSet<>(environment.keySet());
         commonKeys.retainAll(defaultEnv.keySet());
         if (commonKeys.isEmpty() == false) {

buildSrc/src/test/groovy/org/elasticsearch/gradle/BuildPluginTests.java

@@ -22,6 +22,9 @@
 import org.gradle.api.GradleException;
 import org.junit.Test;
 
+import java.net.URI;
+import java.net.URISyntaxException;
+
 
 public class BuildPluginTests extends GradleUnitTestCase {
 
@@ -36,4 +39,25 @@ public void testFailingDockerVersions() {
         BuildPlugin.checkDockerVersionRecent("Docker version 17.04.0, build e68fc7a");
     }
 
+    @Test(expected = GradleException.class)
+    public void testRepositoryURIThatUsesHttpScheme() throws URISyntaxException {
+        final URI uri = new URI("http://s3.amazonaws.com/artifacts.elastic.co/maven");
+        BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
+    }
+
+    public void testRepositoryThatUsesFileScheme() throws URISyntaxException {
+        final URI uri = new URI("file:/tmp/maven");
+        BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
+    }
+
+    public void testRepositoryURIThatUsesHttpsScheme() throws URISyntaxException {
+        final URI uri = new URI("https://s3.amazonaws.com/artifacts.elastic.co/maven");
+        BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
+    }
+
+    public void testRepositoryURIThatUsesS3Scheme() throws URISyntaxException {
+        final URI uri = new URI("s3://artifacts.elastic.co/maven");
+        BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
+    }
+
 }

buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginIT.java

@@ -18,19 +18,29 @@
  */
 package org.elasticsearch.gradle;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.List;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 public class BuildPluginIT extends GradleIntegrationTestCase {
 
+    @Rule
+    public TemporaryFolder tmpDir = new TemporaryFolder();
+
     public void testPluginCanBeApplied() {
         BuildResult result = getGradleRunner("elasticsearch.build")
             .withArguments("hello", "-s")
@@ -46,6 +56,51 @@ public void testCheckTask() {
         assertTaskSuccessful(result, ":check");
     }
 
+    public void testInsecureMavenRepository() throws IOException {
+        final String name = "elastic-maven";
+        final String url = "http://s3.amazonaws.com/artifacts.elastic.co/maven";
+        // add an insecure maven repository to the build.gradle
+        final List<String> lines = Arrays.asList(
+            "repositories {",
+            "  maven {",
+            "    name \"elastic-maven\"",
+            "    url \"" +  url + "\"\n",
+            "  }",
+            "}");
+        runInsecureArtifactRepositoryTest(name, url, lines);
+    }
+
+    public void testInsecureIvyRepository() throws IOException {
+        final String name = "elastic-ivy";
+        final String url = "http://s3.amazonaws.com/artifacts.elastic.co/ivy";
+        // add an insecure ivy repository to the build.gradle
+        final List<String> lines = Arrays.asList(
+            "repositories {",
+            "  ivy {",
+            "    name \"elastic-ivy\"",
+            "    url \"" +  url + "\"\n",
+            "  }",
+            "}");
+        runInsecureArtifactRepositoryTest(name, url, lines);
+    }
+
+    private void runInsecureArtifactRepositoryTest(final String name, final String url, final List<String> lines) throws IOException {
+        final File projectDir = getProjectDir("elasticsearch.build");
+        FileUtils.copyDirectory(projectDir, tmpDir.getRoot(), pathname -> pathname.getPath().contains("/build/") == false);
+        final List<String> buildGradleLines =
+            Files.readAllLines(tmpDir.getRoot().toPath().resolve("build.gradle"), StandardCharsets.UTF_8);
+        buildGradleLines.addAll(lines);
+        Files.write(tmpDir.getRoot().toPath().resolve("build.gradle"), buildGradleLines, StandardCharsets.UTF_8);
+        final BuildResult result = GradleRunner.create()
+            .withProjectDir(tmpDir.getRoot())
+            .withArguments("clean", "hello", "-s", "-i", "--warning-mode=all", "--scan")
+            .withPluginClasspath()
+            .buildAndFail();
+        assertOutputContains(
+            result.getOutput(),
+            "repository [" + name + "] on project with path [:] is not using a secure protocol for artifacts on [" + url + "]");
+    }
+
     public void testLicenseAndNotice() throws IOException {
         BuildResult result = getGradleRunner("elasticsearch.build")
             .withArguments("clean", "assemble", "-s", "-Dlocal.repo.path=" + getLocalTestRepoPath())

buildSrc/version.properties

@@ -21,7 +21,7 @@ slf4j             = 1.6.2
 jna               = 4.5.1
 
 netty             = 4.1.38.Final
-joda              = 2.10.2
+joda              = 2.10.3
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)

client/rest-high-level/build.gradle

@@ -78,6 +78,7 @@ processTestResources {
   from({ zipTree(configurations.restSpec.singleFile) }) {
     include 'rest-api-spec/api/**'
   }
+  from(project(':client:rest-high-level').file('src/test/resources'))
 }
 
 dependencyLicenses {
@@ -96,6 +97,7 @@ forbiddenApisMain {
 }
 File nodeCert = file("./testnode.crt")
 File nodeTrustStore = file("./testnode.jks")
+File pkiTrustCert = file("./src/test/resources/org/elasticsearch/client/security/delegate_pki/testRootCA.crt")
 
 integTest.runner {
   systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
@@ -113,11 +115,18 @@ testClusters.integTest {
   // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API
   setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
+  setting 'xpack.security.authc.realms.file.default_file.order', '0'
+  setting 'xpack.security.authc.realms.native.default_native.order', '1'
+  setting 'xpack.security.authc.realms.pki.pki1.order', '2'
+  setting 'xpack.security.authc.realms.pki.pki1.certificate_authorities', '[ "testRootCA.crt" ]'
+  setting 'xpack.security.authc.realms.pki.pki1.delegation.enabled', 'true'
+
   setting 'indices.lifecycle.poll_interval', '1000ms'
   keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
   user username:  System.getProperty('tests.rest.cluster.username', 'test_user'),
           password:  System.getProperty('tests.rest.cluster.password', 'test-password')
 
   extraConfigFile nodeCert.name, nodeCert
   extraConfigFile nodeTrustStore.name, nodeTrustStore
+  extraConfigFile pkiTrustCert.name, pkiTrustCert
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java

@@ -31,6 +31,8 @@
 import org.elasticsearch.client.security.CreateApiKeyResponse;
 import org.elasticsearch.client.security.CreateTokenRequest;
 import org.elasticsearch.client.security.CreateTokenResponse;
+import org.elasticsearch.client.security.DelegatePkiAuthenticationRequest;
+import org.elasticsearch.client.security.DelegatePkiAuthenticationResponse;
 import org.elasticsearch.client.security.DeletePrivilegesRequest;
 import org.elasticsearch.client.security.DeletePrivilegesResponse;
 import org.elasticsearch.client.security.DeleteRoleMappingRequest;
@@ -969,4 +971,38 @@ public void invalidateApiKeyAsync(final InvalidateApiKeyRequest request, final R
         restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::invalidateApiKey, options,
                 InvalidateApiKeyResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Get an Elasticsearch access token from an {@code X509Certificate} chain. The certificate chain is that of the client from a mutually
+     * authenticated TLS session, and it is validated by the PKI realms with {@code delegation.enabled} toggled to {@code true}.<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delegate-pki-authentication.html"> the
+     * docs</a> for more details.
+     * 
+     * @param request the request containing the certificate chain
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response from the delegate-pki-authentication API key call
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public DelegatePkiAuthenticationResponse delegatePkiAuthentication(DelegatePkiAuthenticationRequest request, RequestOptions options)
+            throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::delegatePkiAuthentication, options,
+                DelegatePkiAuthenticationResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously get an Elasticsearch access token from an {@code X509Certificate} chain. The certificate chain is that of the client
+     * from a mutually authenticated TLS session, and it is validated by the PKI realms with {@code delegation.enabled} toggled to
+     * {@code true}.<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delegate-pki-authentication.html"> the
+     * docs</a> for more details.
+     * 
+     * @param request the request containing the certificate chain
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void delegatePkiAuthenticationAsync(DelegatePkiAuthenticationRequest request, RequestOptions options,
+            ActionListener<DelegatePkiAuthenticationResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::delegatePkiAuthentication, options,
+                DelegatePkiAuthenticationResponse::fromXContent, listener, emptySet());
+    }
 }