Merge remote-tracking branch 'elastic/master' into eventually-consistent-mock-repo

Author: original-brownbear

.ci/init.gradle

@@ -6,6 +6,7 @@ if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFA
         settings.pluginManagement {
             repositories {
                 maven {
+                    name "artifactory-gradle-plugins"
                     url "https://artifactory.elstc.co/artifactory/gradle-plugins"
                     credentials {
                         username System.env.ELASTIC_ARTIFACTORY_USERNAME
@@ -21,6 +22,7 @@ if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFA
             buildscript {
                 repositories {
                     maven {
+                        name "artifactory-gradle-release"
                         url "https://artifactory.elstc.co/artifactory/gradle-release/"
                         credentials {
                             username System.env.ELASTIC_ARTIFACTORY_USERNAME
@@ -31,6 +33,7 @@ if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFA
             }
             repositories {
                 maven {
+                    name "artifactory-gradle-release"
                     url "https://artifactory.elstc.co/artifactory/gradle-release/"
                     credentials {
                         username System.env.ELASTIC_ARTIFACTORY_USERNAME

TESTING.asciidoc

@@ -374,6 +374,7 @@ These are the linux flavors supported, all of which we provide images for
 * debian-9 aka stretch, the current debian stable distribution
 * centos-6
 * centos-7
+* rhel-8
 * fedora-28
 * fedora-29
 * oel-6 aka Oracle Enterprise Linux 6

Vagrantfile

@@ -124,6 +124,12 @@ Vagrant.configure(2) do |config|
       sles_common config, box
     end
   end
+  'rhel-8'.tap do |box|
+    config.vm.define box, define_opts do |config|
+      config.vm.box = 'elastic/rhel-8-x86_64'
+      rpm_common config, box
+    end
+  end
 
   windows_2012r2_box = ENV['VAGRANT_WINDOWS_2012R2_BOX']
   if windows_2012r2_box && windows_2012r2_box.empty? == false
@@ -244,7 +250,7 @@ def linux_common(config,
   SHELL
 
   config.vm.provision 'jdk-11', type: 'shell', inline: <<-SHELL
-    curl -sSL https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz | tar xz -C /opt/
+    curl -sSL https://download.oracle.com/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz | tar xz -C /opt/
   SHELL
 
   # This prevents leftovers from previous tests using the
@@ -405,7 +411,7 @@ def windows_common(config, name)
 
   config.vm.provision 'windows-jdk-11', type: 'shell', inline: <<-SHELL
     New-Item -ItemType Directory -Force -Path "C:/java"
-    Invoke-WebRequest "https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_windows-x64_bin.zip" -OutFile "C:/java/jdk-11.zip"
+    Invoke-WebRequest "https://download.oracle.com/java/GA/jdk11/9/GPL/openjdk-11.0.2_windows-x64_bin.zip" -OutFile "C:/java/jdk-11.zip"
     Expand-Archive -Path "C:/java/jdk-11.zip" -DestinationPath "C:/java/"
   SHELL
 

build.gradle

@@ -88,7 +88,7 @@ subprojects {
       }
       repositories {
         maven {
-          name = 'localTest'
+          name = 'test'
           url = "${rootProject.buildDir}/local-test-repo"
         }
       }
@@ -162,8 +162,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/41426" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

@@ -189,7 +189,7 @@ if (project != rootProject) {
   task integTest(type: Test) {
     // integration test requires the local testing repo for example plugin builds
     dependsOn project.rootProject.allprojects.collect {
-      it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
+      it.tasks.matching { it.name == 'publishNebulaPublicationToTestRepository'}
     }
     dependsOn setupLocalDownloads
     exclude "**/*Tests.class"

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -39,6 +39,9 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.artifacts.repositories.ArtifactRepository
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository
+import org.gradle.api.artifacts.repositories.MavenArtifactRepository
 import org.gradle.api.credentials.HttpHeaderCredentials
 import org.gradle.api.execution.TaskActionListener
 import org.gradle.api.execution.TaskExecutionGraph
@@ -580,6 +583,16 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Adds repositories used by ES dependencies */
     static void configureRepositories(Project project) {
+        project.getRepositories().all { repository ->
+            if (repository instanceof MavenArtifactRepository) {
+                final MavenArtifactRepository maven = (MavenArtifactRepository) repository
+                assertRepositoryURIUsesHttps(maven, project, maven.getUrl())
+                repository.getArtifactUrls().each { uri -> assertRepositoryURIUsesHttps(project, uri) }
+            } else if (repository instanceof IvyArtifactRepository) {
+                final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
+                assertRepositoryURIUsesHttps(ivy, project, ivy.getUrl())
+            }
+        }
         RepositoryHandler repos = project.repositories
         if (System.getProperty("repos.mavenLocal") != null) {
             // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
@@ -589,6 +602,7 @@ class BuildPlugin implements Plugin<Project> {
         }
         repos.jcenter()
         repos.ivy {
+            name "elasticsearch"
             url "https://artifacts.elastic.co/downloads"
             patternLayout {
                 artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]"
@@ -617,6 +631,12 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
+    private static void assertRepositoryURIUsesHttps(final ArtifactRepository repository, final Project project, final URI uri) {
+        if (uri != null && uri.toURL().getProtocol().equals("http")) {
+            throw new GradleException("repository [${repository.name}] on project with path [${project.path}] is using http for artifacts on [${uri.toURL()}]")
+        }
+    }
+
     /**
      * Returns a closure which can be used with a MavenPom for fixing problems with gradle generated poms.
      *

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -89,6 +89,7 @@ class PluginBuildPlugin extends BuildPlugin {
                 project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName ->
                     // Auto add dependent modules to the test cluster
                     if (project.findProject(":modules:${pluginName}") != null) {
+                        project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin)
                         project.testClusters.integTest.module(
                                 project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile)
                         )
@@ -98,9 +99,8 @@ class PluginBuildPlugin extends BuildPlugin {
 
             project.tasks.run.dependsOn(project.tasks.bundlePlugin)
             if (isModule) {
-                project.tasks.run.clusterConfig.module(project)
                 project.tasks.run.clusterConfig.distribution = System.getProperty(
-                        'run.distribution', 'integ-test-zip'
+                        'run.distribution', isXPackModule ? 'default' : 'oss'
                 )
             } else {
                 project.tasks.run.clusterConfig.plugin(project.path)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -936,6 +936,8 @@ class ClusterFormationTasks {
             }
             doLast {
                 project.delete(node.pidFile)
+                // Large tests can exhaust disk space, clean up jdk from the distribution to save some space
+                project.delete(new File(node.homeDir, "jdk"))
             }
         }
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -120,7 +120,6 @@ class RestIntegTestTask extends DefaultTask {
             if (usesTestclusters == true) {
                 ElasticsearchCluster cluster = project.testClusters."${name}"
                 nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }")
-                nonInputProperties.systemProperty('tests.config.dir', "${-> cluster.singleNode().getConfigDir() }")
                 nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }")
             } else {
                 // we pass all nodes to the rest cluster to allow the clients to round-robin between them

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -30,6 +30,7 @@ class VagrantTestPlugin implements Plugin<Project> {
             'oel-6',
             'oel-7',
             'opensuse-42',
+            'rhel-8',
             'sles-12',
             'ubuntu-1604',
             'ubuntu-1804'
@@ -173,6 +174,7 @@ class VagrantTestPlugin implements Plugin<Project> {
           which should work for 5.0.0+. This isn't a real ivy repository but gradle
           is fine with that */
         repos.ivy {
+            name "elasticsearch"
             artifactPattern "https://artifacts.elastic.co/downloads/elasticsearch/[module]-[revision].[ext]"
         }
     }