Merge branch 'master' into pr/30320

Author: Christoph Büscher

Vagrantfile

@@ -31,6 +31,9 @@ Vagrant.configure(2) do |config|
     # Give the box more memory and cpu because our tests are beasts!
     vbox.memory = Integer(ENV['VAGRANT_MEMORY'] || 8192)
     vbox.cpus = Integer(ENV['VAGRANT_CPUS'] || 4)
+
+    # see https://github.com/hashicorp/vagrant/issues/9524
+    vbox.customize ["modifyvm", :id, "--audio", "none"]
   end
 
   # Switch the default share for the project root from /vagrant to

build.gradle

@@ -53,9 +53,23 @@ subprojects {
   description = "Elasticsearch subproject ${project.path}"
 }
 
+apply plugin: 'nebula.info-scm'
+String licenseCommit
+if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
+  licenseCommit = scminfo.change ?: "master" // leniency for non git builds
+} else {
+  licenseCommit = "v${version}"
+}
+String elasticLicenseUrl = "https://raw.githubusercontent.com/elastic/elasticsearch/${licenseCommit}/licenses/ELASTIC-LICENSE.txt"
+
 subprojects {
+  // Default to the apache license
   project.ext.licenseName = 'The Apache Software License, Version 2.0'
   project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+
+  // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
+  project.ext.elasticLicenseUrl = elasticLicenseUrl
+
   // we only use maven publish to add tasks for pom generation
   plugins.withType(MavenPublishPlugin).whenPluginAdded {
     publishing {
@@ -131,7 +145,7 @@ task verifyVersions {
     new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
         xml = new XmlParser().parse(s)
     }
-    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })
+    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }.collect { Version.fromString(it) })
 
     // Limit the known versions to those that should be index compatible, and are not future versions
     knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
@@ -205,9 +219,9 @@ subprojects {
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':server',
-    "org.elasticsearch:elasticsearch-cli:${version}": ':server:cli',
-    "org.elasticsearch:elasticsearch-core:${version}": ':libs:elasticsearch-core',
-    "org.elasticsearch:elasticsearch-nio:${version}": ':libs:elasticsearch-nio',
+    "org.elasticsearch:elasticsearch-cli:${version}": ':libs:cli',
+    "org.elasticsearch:elasticsearch-core:${version}": ':libs:core',
+    "org.elasticsearch:elasticsearch-nio:${version}": ':libs:nio',
     "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
     "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
@@ -543,7 +557,7 @@ subprojects { project ->
   }
 }
 
-/* Remove assemble on all qa projects because we don't need to publish
+/* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish
  * artifacts for them. */
 gradle.projectsEvaluated {
   subprojects {
@@ -553,6 +567,11 @@ gradle.projectsEvaluated {
         project.tasks.remove(assemble)
         project.build.dependsOn.remove('assemble')
       }
+      Task dependenciesInfo = project.tasks.findByName('dependenciesInfo')
+      if (dependenciesInfo) {
+        project.tasks.remove(dependenciesInfo)
+        project.precommit.dependsOn.remove('dependenciesInfo')
+      }
     }
   }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -762,6 +762,10 @@ class BuildPlugin implements Plugin<Project> {
 
     private static configureDependenciesInfo(Project project) {
         Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class)
-        deps.dependencies = project.configurations.compile.allDependencies
+        deps.runtimeConfiguration = project.configurations.runtime
+        deps.compileOnlyConfiguration = project.configurations.compileOnly
+        project.afterEvaluate {
+            deps.mappings = project.dependencyLicenses.mappings
+        }
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy

@@ -19,14 +19,19 @@
 
 package org.elasticsearch.gradle
 
+import org.elasticsearch.gradle.precommit.DependencyLicensesTask
 import org.gradle.api.DefaultTask
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.Dependency
+import org.gradle.api.artifacts.DependencyResolutionListener
 import org.gradle.api.artifacts.DependencySet
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.TaskAction
 
+import java.util.regex.Matcher
+import java.util.regex.Pattern
 
 /**
  * A task to gather information about the dependencies and export them into a csv file.
@@ -44,7 +49,14 @@ public class DependenciesInfoTask extends DefaultTask {
 
     /** Dependencies to gather information from. */
     @Input
-    public DependencySet dependencies
+    public Configuration runtimeConfiguration
+
+    /** We subtract compile-only dependencies. */
+    @Input
+    public Configuration compileOnlyConfiguration
+
+    @Input
+    public LinkedHashMap<String, String> mappings
 
     /** Directory to read license files */
     @InputDirectory
@@ -59,15 +71,34 @@ public class DependenciesInfoTask extends DefaultTask {
 
     @TaskAction
     public void generateDependenciesInfo() {
+
+        final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies()
+        // we have to resolve the transitive dependencies and create a group:artifactId:version map
+        final Set<String> compileOnlyArtifacts =
+                compileOnlyConfiguration
+                        .getResolvedConfiguration()
+                        .resolvedArtifacts
+                        .collect { it -> "${it.moduleVersion.id.group}:${it.moduleVersion.id.name}:${it.moduleVersion.id.version}" }
+
         final StringBuilder output = new StringBuilder()
 
-        for (Dependency dependency : dependencies) {
-            // Only external dependencies are checked
-            if (dependency.group != null && dependency.group.contains("elasticsearch") == false) {
-                final String url = createURL(dependency.group, dependency.name, dependency.version)
-                final String licenseType = getLicenseType(dependency.group, dependency.name)
-                output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
+        for (final Dependency dependency : runtimeDependencies) {
+            // we do not need compile-only dependencies here
+            if (compileOnlyArtifacts.contains("${dependency.group}:${dependency.name}:${dependency.version}")) {
+                continue
             }
+            // only external dependencies are checked
+            if (dependency.group != null && dependency.group.contains("org.elasticsearch")) {
+                continue
+            }
+
+            final String url = createURL(dependency.group, dependency.name, dependency.version)
+            final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, dependency.name)
+            logger.info("mapped dependency ${dependency.group}:${dependency.name} to ${dependencyName} for license info")
+
+            final String licenseType = getLicenseType(dependency.group, dependencyName)
+            output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
+
         }
         outputFile.setText(output.toString(), 'UTF-8')
     }
@@ -109,7 +140,8 @@ public class DependenciesInfoTask extends DefaultTask {
         }
 
         if (license) {
-            final String content = license.readLines("UTF-8").toString()
+            // replace * because they are sometimes used at the beginning lines as if the license was a multi-line comment
+            final String content = new String(license.readBytes(), "UTF-8").replaceAll("\\s+", " ").replaceAll("\\*", " ")
             final String spdx = checkSPDXLicense(content)
             if (spdx ==  null) {
                 // License has not be identified as SPDX.
@@ -133,15 +165,88 @@ public class DependenciesInfoTask extends DefaultTask {
     private String checkSPDXLicense(final String licenseText) {
         String spdx = null
 
-        final String APACHE_2_0 = "Apache.*License.*(v|V)ersion 2.0"
-        final String BSD_2 = "BSD 2-clause.*License"
+        final String APACHE_2_0 = "Apache.*License.*(v|V)ersion.*2\\.0"
+
+        final String BSD_2 = """
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1\\. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer\\.
+ 2\\. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution\\.
+
+THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\.
+IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\.
+""".replaceAll("\\s+", "\\\\s*")
+
+        final String BSD_3 = """
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ (1\\.)? Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer\\.
+ (2\\.)? Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution\\.
+ ((3\\.)? The name of .+ may not be used to endorse or promote products
+    derived from this software without specific prior written permission\\.|
+  (3\\.)? Neither the name of .+ nor the names of its
+     contributors may be used to endorse or promote products derived from
+     this software without specific prior written permission\\.)
+
+THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\.
+IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\.
+""".replaceAll("\\s+", "\\\\s*")
+
         final String CDDL_1_0 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.0"
         final String CDDL_1_1 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.1"
         final String ICU = "ICU License - ICU 1.8.1 and later"
         final String LGPL_3 = "GNU LESSER GENERAL PUBLIC LICENSE.*Version 3"
-        final String MIT = "MIT License"
+
+        final String MIT = """
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files \\(the "Software"\\), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software\\.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\\. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE\\.
+""".replaceAll("\\s+", "\\\\s*")
+
         final String MOZILLA_1_1 = "Mozilla Public License.*Version 1.1"
 
+        final String MOZILLA_2_0 = "Mozilla\\s*Public\\s*License\\s*Version\\s*2\\.0"
+
         switch (licenseText) {
             case ~/.*${APACHE_2_0}.*/:
                 spdx = 'Apache-2.0'
@@ -152,6 +257,9 @@ public class DependenciesInfoTask extends DefaultTask {
             case ~/.*${BSD_2}.*/:
                 spdx = 'BSD-2-Clause'
                 break
+            case ~/.*${BSD_3}.*/:
+                spdx = 'BSD-3-Clause'
+                break
             case ~/.*${LGPL_3}.*/:
                 spdx = 'LGPL-3.0'
                 break
@@ -167,6 +275,9 @@ public class DependenciesInfoTask extends DefaultTask {
             case ~/.*${MOZILLA_1_1}.*/:
                 spdx = 'MPL-1.1'
                 break
+            case ~/.*${MOZILLA_2_0}.*/:
+                spdx = 'MPL-2.0'
+                break
             default:
                 break
         }

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -32,6 +32,8 @@ public class DocsTestPlugin extends RestTestPlugin {
     public void apply(Project project) {
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
+        // The distribution can be configured with -Dtests.distribution on the command line
+        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'zip')
         // Docs are published separately so no need to assemble
         project.tasks.remove(project.assemble)
         project.build.dependsOn.remove('assemble')
@@ -43,6 +45,8 @@ public class DocsTestPlugin extends RestTestPlugin {
             '\\{version\\}':
                 VersionProperties.elasticsearch.toString().replace('-SNAPSHOT', ''),
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
+            '\\{build_flavor\\}' :
+                project.integTestCluster.distribution.startsWith('oss-') ? 'oss' : 'default',
         ]
         Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
         listSnippets.group 'Docs'

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -27,7 +27,6 @@ import org.gradle.api.tasks.OutputDirectory
 
 import java.nio.file.Files
 import java.nio.file.Path
-import java.util.regex.Matcher
 
 /**
  * Generates REST tests for each snippet marked // TEST.
@@ -100,6 +99,14 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         return snippet.language == 'js' || snippet.curl
     }
 
+    /**
+     * Certain requests should not have the shard failure check because the
+     * format of the response is incompatible i.e. it is not a JSON object.
+     */
+    static shouldAddShardFailureCheck(String path) {
+        return path.startsWith('_cat') == false &&  path.startsWith('_xpack/ml/datafeeds/') == false
+    }
+
     /**
      * Converts Kibana's block quoted strings into standard JSON. These
      * {@code """} delimited strings can be embedded in CONSOLE and can
@@ -309,13 +316,11 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
              * no shard succeeds. But we need to fail the tests on all of these
              * because they mean invalid syntax or broken queries or something
              * else that we don't want to teach people to do. The REST test
-             * framework doesn't allow us to has assertions in the setup
-             * section so we have to skip it there. We also have to skip _cat
-             * actions because they don't return json so we can't is_false
-             * them. That is ok because they don't have this
-             * partial-success-is-success thing.
+             * framework doesn't allow us to have assertions in the setup
+             * section so we have to skip it there. We also omit the assertion
+             * from APIs that don't return a JSON object
              */
-            if (false == inSetup && false == path.startsWith('_cat')) {
+            if (false == inSetup && shouldAddShardFailureCheck(path)) {
                 current.println("  - is_false: _shards.failures")
             }
         }