Verify signatures on official plugins (#30800)

We sign our official plugins yet this is not well-advertised and not at
all consumed during plugin installation. For plugins that are installed
over the intertubes, verifying that the downloaded artifact is signed by
our signing key would establish both integrity and validity of the
downloaded artifact. The chain of trust here is simple: our installable
artifacts (archive and package distributions) so that if a user trusts
our packages via their signatures, and our plugin installer (which would
be executing trusted code) verifies the downloaded plugin, then the user
can trust the downloaded plugin too. This commit adds verification of
official plugins downloaded during installation. We do not add
verification for offline plugin installs; a user can download our
signatures and verify the artifacts themselves.

This commit also needs to solve a few interesting challenges. One of
these is that we want the bouncy castle JARs on the classpath only for
the plugin installer, but not for the runtime
Elasticsearch. Additionally, we want these JARs to not be present for
the JAR hell checks. To address this, we shift these JARs into a
sub-directory of lib (lib/tools/plugin-cli) that is only loaded for the
plugin installer, and in the plugin installer we filter any JARs in this
directory from the JAR hell check.

Author: jasontedor

distribution/archives/build.gradle

@@ -49,7 +49,9 @@ task createPluginsDir(type: EmptyDirTask) {
 CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean oss) {
   return copySpec {
     into("elasticsearch-${version}") {
-      with libFiles
+      into('lib') {
+        with libFiles
+      }
       into('config') {
         dirMode 0750
         fileMode 0660

distribution/build.gradle

@@ -227,13 +227,15 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
      *                   Common files in all distributions                       *
      *****************************************************************************/
     libFiles = copySpec {
-      into 'lib'
+      // delay by using closures, since they have not yet been configured, so no jar task exists yet
       from { project(':server').jar }
       from { project(':server').configurations.runtime }
       from { project(':libs:plugin-classloader').jar }
-      // delay add tools using closures, since they have not yet been configured, so no jar task exists yet
       from { project(':distribution:tools:launchers').jar }
-      from { project(':distribution:tools:plugin-cli').jar }
+      into('tools/plugin-cli') {
+        from { project(':distribution:tools:plugin-cli').jar }
+        from { project(':distribution:tools:plugin-cli').configurations.runtime }
+      }
     }
 
     modulesFiles = { oss ->

distribution/packages/build.gradle

@@ -124,13 +124,23 @@ Closure commonPackageConfig(String type, boolean oss) {
         include 'README.textile'
         fileMode 0644
       }
+      into('lib') {
+        with copySpec {
+          with libFiles
+          // we need to specify every intermediate directory so we iterate through the parents; duplicate calls with the same part are fine
+          eachFile { FileCopyDetails fcp ->
+            String[] segments = fcp.relativePath.segments
+            for (int i = segments.length - 2; i > 0 && segments[i] != 'lib'; --i) {
+              directory('/' + segments[0..i].join('/'), 0755)
+            }
+            fcp.mode = 0644
+          }
+        }
+      }
       into('modules') {
         with copySpec {
           with modulesFiles(oss)
-          // we need to specify every intermediate directory, but modules could have sub directories
-          // and there might not be any files as direct children of intermediates (eg platform)
-          // so we must iterate through the parents, but duplicate calls with the same path
-          // are ok (they don't show up in the built packages)
+          // we need to specify every intermediate directory so we iterate through the parents; duplicate calls with the same part are fine
           eachFile { FileCopyDetails fcp ->
             String[] segments = fcp.relativePath.segments
             for (int i = segments.length - 2; i > 0 && segments[i] != 'modules'; --i) {
@@ -251,8 +261,8 @@ ospackage {
     signingKeyId = project.hasProperty('signing.keyId') ? project.property('signing.keyId') : 'D88E42B4'
     signingKeyPassphrase = project.property('signing.password')
     signingKeyRingFile = project.hasProperty('signing.secretKeyRingFile') ?
-                         project.file(project.property('signing.secretKeyRingFile')) :
-                         new File(new File(System.getProperty('user.home'), '.gnupg'), 'secring.gpg')
+            project.file(project.property('signing.secretKeyRingFile')) :
+            new File(new File(System.getProperty('user.home'), '.gnupg'), 'secring.gpg')
   }
 
   requires('coreutils')
@@ -263,7 +273,6 @@ ospackage {
   permissionGroup 'root'
 
   into '/usr/share/elasticsearch'
-  with libFiles
   with noticeFile
 }
 

distribution/src/bin/elasticsearch-cli

@@ -10,6 +10,12 @@ do
   source "`dirname "$0"`"/$additional_source
 done
 
+IFS=';' read -r -a additional_classpath_directories <<< "$ES_ADDITIONAL_CLASSPATH_DIRECTORIES"
+for additional_classpath_directory in "${additional_classpath_directories[@]}"
+do
+  ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/$additional_classpath_directory/*"
+done
+
 exec \
   "$JAVA" \
   $ES_JAVA_OPTS \

distribution/src/bin/elasticsearch-cli.bat

@@ -11,6 +11,12 @@ for /f "tokens=1*" %%a in ("%*") do (
   set arguments=%%b
 )
 
+if defined ES_ADDITIONAL_CLASSPATH_DIRECTORIES (
+  for %%a in ("%ES_ADDITIONAL_CLASSPATH_DIRECTORIES:;=","%") do (
+    set ES_CLASSPATH=!ES_CLASSPATH!;!ES_HOME!/%%a/*
+  )
+)
+
 %JAVA% ^
   %ES_JAVA_OPTS% ^
   -Des.path.home="%ES_HOME%" ^

distribution/src/bin/elasticsearch-plugin

@@ -1,5 +1,6 @@
 #!/bin/bash
 
-"`dirname "$0"`"/elasticsearch-cli \
+ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/plugin-cli \
+  "`dirname "$0"`"/elasticsearch-cli \
   org.elasticsearch.plugins.PluginCli \
   "$@"

distribution/src/bin/elasticsearch-plugin.bat

@@ -3,6 +3,7 @@
 setlocal enabledelayedexpansion
 setlocal enableextensions
 
+set ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/plugin-cli
 call "%~dp0elasticsearch-cli.bat" ^
   org.elasticsearch.plugins.PluginCli ^
   %* ^

distribution/tools/plugin-cli/build.gradle

@@ -19,14 +19,22 @@
 
 apply plugin: 'elasticsearch.build'
 
+archivesBaseName = 'elasticsearch-plugin-cli'
+
 dependencies {
   compileOnly "org.elasticsearch:elasticsearch:${version}"
   compileOnly "org.elasticsearch:elasticsearch-cli:${version}"
+  compile "org.bouncycastle:bcpg-jdk15on:1.59"
+  compile "org.bouncycastle:bcprov-jdk15on:1.59"
   testCompile "org.elasticsearch.test:framework:${version}"
   testCompile 'com.google.jimfs:jimfs:1.1'
   testCompile 'com.google.guava:guava:18.0'
 }
 
+dependencyLicenses {
+  mapping from: /bc.*/, to: 'bouncycastle'
+}
+
 test {
   // TODO: find a way to add permissions for the tests in this module
   systemProperty 'tests.security.manager', 'false'

distribution/tools/plugin-cli/licenses/bcpg-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+ee93e5376bb6cf0a15c027b5f5e4393f2738e709

distribution/tools/plugin-cli/licenses/bcprov-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+2507204241ab450456bdb8e8c0a8f986e418bd99