Update Format, add new settings into the setting test

Author: Xu Zhang

plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java

@@ -154,6 +154,5 @@ interface CLOUD_S3 {
         Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
     }
 
-    AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
     AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials);
 }

plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java

@@ -62,12 +62,9 @@ public InternalAwsS3Service(Settings settings) {
     }
 
     @Override
-    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
-        return client(endpoint, protocol, region, account, key, maxRetries, null);
-    }
+    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
+                                        EncryptionMaterials clientSideEncryptionMaterials) {
 
-    @Override
-    public AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) {
         if (Strings.isNullOrEmpty(endpoint)) {
             // We need to set the endpoint based on the region
             if (region != null) {
@@ -144,8 +141,7 @@ private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, Stri
                     credentials,
                     encryptionMaterialsProvider,
                     clientConfiguration,
-                    cryptoConfiguration
-            );
+                    cryptoConfiguration);
         } else {
             client = new AmazonS3Client(credentials, clientConfiguration);
         }

plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java

@@ -20,8 +20,6 @@
 package org.elasticsearch.cloud.aws.blobstore;
 
 import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.AmazonS3EncryptionClient;
-import com.amazonaws.services.s3.Headers;
 import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;

plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java

@@ -147,7 +147,6 @@ public void onModule(SettingsModule settingsModule) {
         settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
         settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
         settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
-
         settingsModule.registerSetting(S3Repository.Repository.CLIENT_PRIVATE_KEY);
         settingsModule.registerSetting(S3Repository.Repository.CLIENT_PUBLIC_KEY);
         settingsModule.registerSetting(S3Repository.Repository.CLIENT_SYMMETRIC_KEY);

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java

@@ -247,7 +247,6 @@ public interface Repository {
          * base_path
          * @see  Repositories#BASE_PATH_SETTING
          */
-
         Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);
 
         /**
@@ -319,15 +318,15 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings,
         String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
         String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
 
-        logger.debug("using bqucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
+        logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
                 bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
 
         String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
         String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
 
         // parse and validate the client side encryption setting
         String symmetricKeyBase64 = getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY);
-        String publicKeyBase64 =getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY);
+        String publicKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY);
         String privateKeyBase64 = getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY);
 
         EncryptionMaterials clientSideEncryptionMaterials = initClientSideEncryption(symmetricKeyBase64, publicKeyBase64, privateKeyBase64, name);
@@ -366,7 +365,7 @@ private EncryptionMaterials initClientSideEncryption(String symmetricKey, String
         EncryptionMaterials clientSideEncryptionMaterials = null;
 
         if (Strings.isNullOrEmpty(symmetricKey) == false && (Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false)) {
-            throw new RepositoryException(name.name(), "Client-side encryption: You can't specify an symmetric key AND a public/private key pair");
+            throw new RepositoryException(name.name(), "Client-side encryption: You can't specify a symmetric key AND a public/private key pair");
         }
 
         if (Strings.isNullOrEmpty(symmetricKey) == false || Strings.isNullOrEmpty(publicKey) == false || Strings.isNullOrEmpty(privateKey) == false) {
@@ -380,7 +379,7 @@ private EncryptionMaterials initClientSideEncryption(String symmetricKey, String
                 if (Strings.isNullOrEmpty(symmetricKey) == false) {
                     clientSideEncryptionMaterials = new EncryptionMaterials(new SecretKeySpec(Base64.decode(symmetricKey), "AES"));
                 } else {
-                    if (Strings.isNullOrEmpty(publicKey)|| Strings.isNullOrEmpty(privateKey)){
+                    if (Strings.isNullOrEmpty(publicKey) || Strings.isNullOrEmpty(privateKey)) {
                         String missingKey = Strings.isNullOrEmpty(publicKey) ? "public key" : "private key";
                         throw new RepositoryException(name.name(), "Client-side encryption: " + missingKey + " is missing");
                     }

plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java

@@ -97,6 +97,7 @@ public class RepositoryS3SettingsTests extends ESTestCase {
         .put(Repository.STORAGE_CLASS_SETTING.getKey(), "repository-class")
         .put(Repository.CANNED_ACL_SETTING.getKey(), "repository-acl")
         .put(Repository.BASE_PATH_SETTING.getKey(), "repository-basepath")
+
         .build();
 
     /**
@@ -125,6 +126,9 @@ public void testRepositorySettingsGlobalOnly() {
         assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
         assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
         assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
+        assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
+        assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
+        assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
     }
 
     /**
@@ -153,6 +157,9 @@ public void testRepositorySettingsGlobalOverloadedByS3() {
         assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
         assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
         assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
+        assertThat(getValue(repositorySettings, Repository.CLIENT_SYMMETRIC_KEY, Repositories.CLIENT_SYMMETRIC_KEY), isEmptyString());
+        assertThat(getValue(repositorySettings, Repository.CLIENT_PRIVATE_KEY, Repositories.CLIENT_PRIVATE_KEY), isEmptyString());
+        assertThat(getValue(repositorySettings, Repository.CLIENT_PUBLIC_KEY, Repositories.CLIENT_PUBLIC_KEY), isEmptyString());
     }
 
     /**
@@ -329,6 +336,7 @@ private Settings buildSettings(Settings... global) {
 
     private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage)
         throws IOException {
+
         Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
         RepositorySettings s3RepositorySettings =  new RepositorySettings(nodeSettings, Settings.builder()
             .put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer)

plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java

@@ -20,6 +20,7 @@
 
 import com.amazonaws.Protocol;
 import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.EncryptionMaterials;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -51,8 +52,8 @@ public TestAwsS3Service(Settings settings) {
 
 
     @Override
-    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
-        return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
+    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries, EncryptionMaterials clientSideEncryptionMaterials) {
+        return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, clientSideEncryptionMaterials));
     }
 
     private AmazonS3 cachedWrapper(AmazonS3 client) {

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java

@@ -104,7 +104,7 @@ public void testSimpleWorkflow() {
         logger.info("-->  creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
         PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
                 .setType("s3").setSettings(settings
-                ).get();
+                        ).get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
         createIndex("test-idx-1", "test-idx-2", "test-idx-3");
@@ -209,6 +209,7 @@ public void testEncryption() {
             S3Repository.Repositories.REGION_SETTING.get(settings),
             S3Repository.Repositories.KEY_SETTING.get(settings),
             S3Repository.Repositories.SECRET_SETTING.get(settings),
+            null,
             null);
 
         String bucketName = bucket.get("bucket");
@@ -553,7 +554,7 @@ public void cleanRepositoryFiles(String basePath) {
             // We check that settings has been set in elasticsearch.yml integration test file
             // as described in README
             assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
-            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null);
+            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null, null);
             try {
                 ObjectListing prevListing = null;
                 //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html