Merge branch 'master' into resync-ops

Author: dnhatn

TESTING.asciidoc

@@ -340,13 +340,13 @@ be downloaded again unless they have been updated to a new version.
 +
 . Run the tests with `./gradlew packagingTest`. This will cause Gradle to build
 the tar, zip, and deb packages and all the plugins. It will then run the tests
-on ubuntu-1404 and centos-7. We chose those two distributions as the default
+on ubuntu-1604 and centos-7. We chose those two distributions as the default
 because they cover deb and rpm packaging and SyvVinit and systemd.
 
 You can choose which boxes to test by setting the `-Pvagrant.boxes` project property. All of
 the valid options for this property are:
 
-* `sample` - The default, only chooses ubuntu-1404 and centos-7
+* `sample` - The default, only chooses ubuntu-1604 and centos-7
 * List of box names, comma separated (e.g. `oel-7,fedora-28`) - Chooses exactly the boxes listed.
 * `linux-all` - All linux boxes.
 * `windows-all` - All Windows boxes. If there are any Windows boxes which do not
@@ -364,11 +364,10 @@ will remain running and you'll have to stop them manually with `./gradlew stop`
 
 All the regular vagrant commands should just work so you can get a shell in a
 VM running trusty by running
-`vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404`.
+`vagrant up ubuntu-1604 --provider virtualbox && vagrant ssh ubuntu-1604`.
 
 These are the linux flavors supported, all of which we provide images for
 
-* ubuntu-1404 aka trusty
 * ubuntu-1604 aka xenial
 * ubuntu-1804 aka bionic beaver
 * debian-8 aka jessie
@@ -422,13 +421,13 @@ It's important to think of VMs like cattle. If they become lame you just shoot
 them and let vagrant reprovision them. Say you've hosed your precise VM:
 
 ----------------------------------------------------
-vagrant ssh ubuntu-1404 -c 'sudo rm -rf /bin'; echo oops
+vagrant ssh ubuntu-1604 -c 'sudo rm -rf /bin'; echo oops
 ----------------------------------------------------
 
 All you've got to do to get another one is
 
 ----------------------------------------------
-vagrant destroy -f ubuntu-1404 && vagrant up ubuntu-1404 --provider virtualbox
+vagrant destroy -f ubuntu-1604 && vagrant up ubuntu-1604 --provider virtualbox
 ----------------------------------------------
 
 The whole process takes a minute and a half on a modern laptop, two and a half

Vagrantfile

@@ -46,12 +46,6 @@ Vagrant.configure(2) do |config|
   PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd
   config.vm.synced_folder PROJECT_DIR, '/project'
 
-  'ubuntu-1404'.tap do |box|
-    config.vm.define box, define_opts do |config|
-      config.vm.box = 'elastic/ubuntu-14.04-x86_64'
-      deb_common config, box
-    end
-  end
   'ubuntu-1604'.tap do |box|
     config.vm.define box, define_opts do |config|
       config.vm.box = 'elastic/ubuntu-16.04-x86_64'

build.gradle

@@ -162,8 +162,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/40319" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

@@ -245,6 +245,7 @@ if (project != rootProject) {
 
   forbiddenPatterns {
     exclude '**/*.wav'
+    exclude '**/*.p12'
     // the file that actually defines nocommit
     exclude '**/ForbiddenPatternsTask.java'
   }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -361,16 +361,8 @@ class BuildPlugin implements Plugin<Project> {
             compilerJavaHome = findJavaHome(compilerJavaProperty)
         }
         if (compilerJavaHome == null) {
-            if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
-                // IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
-                return Jvm.current().javaHome
-            } else {
-                throw new GradleException(
-                        "JAVA_HOME must be set to build Elasticsearch. " +
-                                "Note that if the variable was just set you might have to run `./gradlew --stop` for " +
-                                "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
-                )
-            }
+            // if JAVA_HOME does not set,so we use the JDK that Gradle was run with.
+            return Jvm.current().javaHome
         }
         return compilerJavaHome
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -31,7 +31,6 @@ class VagrantTestPlugin implements Plugin<Project> {
             'oel-7',
             'opensuse-42',
             'sles-12',
-            'ubuntu-1404',
             'ubuntu-1604',
             'ubuntu-1804'
     ])
@@ -48,7 +47,7 @@ class VagrantTestPlugin implements Plugin<Project> {
     /** Boxes used when sampling the tests **/
     static final List<String> SAMPLE = unmodifiableList([
             'centos-7',
-            'ubuntu-1404'
+            'ubuntu-1604'
     ])
 
     /** All distributions to bring into test VM, whether or not they are used **/

buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java

@@ -0,0 +1,233 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.http;
+
+import org.gradle.api.logging.Logger;
+import org.gradle.api.logging.Logging;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateFactory;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A utility to wait for a specific HTTP resource to be available, optionally with customized TLS trusted CAs.
+ * This is logically similar to using the Ant Get task to retrieve a resource, but with the difference that it can
+ * access resources that do not use the JRE's default trusted CAs.
+ */
+public class WaitForHttpResource {
+
+    private static final Logger logger = Logging.getLogger(WaitForHttpResource.class);
+
+    private Set<Integer> validResponseCodes = Collections.singleton(200);
+    private URL url;
+    private Set<File> certificateAuthorities;
+    private File trustStoreFile;
+    private String trustStorePassword;
+    private String username;
+    private String password;
+
+    public WaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
+        this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
+    }
+
+    public WaitForHttpResource(URL url) {
+        this.url = url;
+    }
+
+    public void setValidResponseCodes(int... validResponseCodes) {
+        this.validResponseCodes = new HashSet<>(validResponseCodes.length);
+        for (int rc : validResponseCodes) {
+            this.validResponseCodes.add(rc);
+        }
+    }
+
+    public void setCertificateAuthorities(File... certificateAuthorities) {
+        this.certificateAuthorities = new HashSet<>(Arrays.asList(certificateAuthorities));
+    }
+
+    public void setTrustStoreFile(File trustStoreFile) {
+        this.trustStoreFile = trustStoreFile;
+    }
+
+    public void setTrustStorePassword(String trustStorePassword) {
+        this.trustStorePassword = trustStorePassword;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public boolean wait(int durationInMs) throws GeneralSecurityException, InterruptedException, IOException {
+        final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs);
+        final long sleep = Long.max(durationInMs / 10, 100);
+
+        final SSLContext ssl;
+        final KeyStore trustStore = buildTrustStore();
+        if (trustStore != null) {
+            ssl = createSslContext(trustStore);
+        } else {
+            ssl = null;
+        }
+        IOException failure = null;
+        for (; ; ) {
+            try {
+                checkResource(ssl);
+                return true;
+            } catch (IOException e) {
+                logger.debug("Failed to access resource [{}]", url, e);
+                failure = e;
+            }
+            if (System.nanoTime() < waitUntil) {
+                Thread.sleep(sleep);
+            } else {
+                logger.error("Failed to access url [{}]", url, failure);
+                return false;
+            }
+        }
+    }
+
+    protected void checkResource(SSLContext ssl) throws IOException {
+        try {
+            final HttpURLConnection connection = buildConnection(ssl);
+            connection.connect();
+            final Integer response = connection.getResponseCode();
+            if (validResponseCodes.contains(response)) {
+                logger.info("Got successful response [{}] from URL [{}]", response, url);
+                return;
+            } else {
+                throw new IOException(response + " " + connection.getResponseMessage());
+            }
+        } catch (IOException e) {
+            throw e;
+        }
+    }
+
+    HttpURLConnection buildConnection(SSLContext ssl) throws IOException {
+        final HttpURLConnection connection = (HttpURLConnection) this.url.openConnection();
+        configureSslContext(connection, ssl);
+        configureBasicAuth(connection);
+        connection.setRequestMethod("GET");
+        return connection;
+    }
+
+    private void configureSslContext(HttpURLConnection connection, SSLContext ssl) {
+        if (ssl != null) {
+            if (connection instanceof HttpsURLConnection) {
+                ((HttpsURLConnection) connection).setSSLSocketFactory(ssl.getSocketFactory());
+            } else {
+                throw new IllegalStateException("SSL trust has been configured, but [" + url + "] is not a 'https' URL");
+            }
+        }
+    }
+
+    private void configureBasicAuth(HttpURLConnection connection) {
+        if (username != null) {
+            if (password == null) {
+                throw new IllegalStateException("Basic Auth user [" + username
+                    + "] has been set, but no password has been configured");
+            }
+            connection.setRequestProperty("Authorization",
+                "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)));
+        }
+    }
+
+    KeyStore buildTrustStore() throws GeneralSecurityException, IOException {
+        if (this.certificateAuthorities != null) {
+            if (trustStoreFile != null) {
+                throw new IllegalStateException("Cannot specify both truststore and CAs");
+            }
+            return buildTrustStoreFromCA();
+        } else if (trustStoreFile != null) {
+            return buildTrustStoreFromFile();
+        } else {
+            return null;
+        }
+    }
+
+    private KeyStore buildTrustStoreFromFile() throws GeneralSecurityException, IOException {
+        KeyStore keyStore = KeyStore.getInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
+        try (InputStream input = new FileInputStream(trustStoreFile)) {
+            keyStore.load(input, trustStorePassword == null ? null : trustStorePassword.toCharArray());
+        }
+        return keyStore;
+    }
+
+    private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOException {
+        final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
+        store.load(null, null);
+        final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
+        int counter = 0;
+        for (File ca : certificateAuthorities) {
+            try (InputStream input = new FileInputStream(ca)) {
+                for (Certificate certificate : certFactory.generateCertificates(input)) {
+                    store.setCertificateEntry("cert-" + counter, certificate);
+                    counter++;
+                }
+            }
+        }
+        return store;
+    }
+
+    private SSLContext createSslContext(KeyStore trustStore) throws GeneralSecurityException {
+        checkForTrustEntry(trustStore);
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+        tmf.init(trustStore);
+        SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
+        sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
+        return sslContext;
+    }
+
+    private void checkForTrustEntry(KeyStore trustStore) throws KeyStoreException {
+        Enumeration<String> enumeration = trustStore.aliases();
+        while (enumeration.hasMoreElements()) {
+            if (trustStore.isCertificateEntry(enumeration.nextElement())) {
+                // found trusted cert entry
+                return;
+            }
+        }
+        throw new IllegalStateException("Trust-store does not contain any trusted certificate entries");
+    }
+}