From 3fb343c6c6575ce0c46f2bfba9b61f167a1ac108 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 26 Jun 2019 13:43:32 +0200 Subject: [PATCH 1/3] Add SAS Token Authentication Support to Azure Repo Plugin (#42982) (#43618) * Added setting for SAS token * Added support for the token in tests * Relates #42117 --- docs/plugins/repository-azure.asciidoc | 10 ++- .../qa/microsoft-azure-storage/build.gradle | 4 +- .../azure/AzureRepositoryPlugin.java | 1 + .../azure/AzureStorageService.java | 4 +- .../azure/AzureStorageSettings.java | 66 +++++++++++-------- .../azure/AzureSettingsParserTests.java | 3 - .../azure/AzureStorageServiceTests.java | 12 +++- 7 files changed, 62 insertions(+), 38 deletions(-) diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index 5ea7ee65e7f01..f30dc4fbda381 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -19,7 +19,11 @@ bin/elasticsearch-keystore add azure.client.default.account bin/elasticsearch-keystore add azure.client.default.key ---------------------------------------------------------------- -Where `account` is the azure account name and `key` the azure secret key. +Where `account` is the azure account name and `key` the azure secret key. Instead of an azure secret key under `key`, you can alternatively +define a shared access signatures (SAS) token under `sas_token` to use for authentication instead. When using an SAS token instead of an +account key, the SAS token must have read (r), write (w), list (l), and delete (d) permissions for the repository base path and +all its contents. These permissions need to be granted for the blob service (b) and apply to resource types service (s), container (c), and +object (o). These settings are used by the repository's internal azure client. Note that you can also define more than one account: @@ -29,14 +33,14 @@ Note that you can also define more than one account: bin/elasticsearch-keystore add azure.client.default.account bin/elasticsearch-keystore add azure.client.default.key bin/elasticsearch-keystore add azure.client.secondary.account -bin/elasticsearch-keystore add azure.client.secondary.key +bin/elasticsearch-keystore add azure.client.secondary.sas_token ---------------------------------------------------------------- `default` is the default account name which will be used by a repository, unless you set an explicit one in the <>. -Both `account` and `key` storage settings are +The `account`, `key`, and `sas_token` storage settings are {ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you reload the settings, the internal azure clients, which are used to transfer the snapshot, will utilize the latest settings from the keystore. diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle index d9658d4d2f9e2..812db5b438676 100644 --- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle +++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle @@ -33,12 +33,14 @@ String azureAccount = System.getenv("azure_storage_account") String azureKey = System.getenv("azure_storage_key") String azureContainer = System.getenv("azure_storage_container") String azureBasePath = System.getenv("azure_storage_base_path") +String azureSasToken = System.getenv("azure_storage_sas_token") -if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) { +if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) { azureAccount = 'azure_integration_test_account' azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64 azureContainer = 'container_test' azureBasePath = 'integration_test' + azureSasToken = '' useFixture = true } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index cd7a2d132c3f8..366e53ca0fc23 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -59,6 +59,7 @@ public List> getSettings() { AzureStorageSettings.Storage.STORAGE_ACCOUNTS, AzureStorageSettings.ACCOUNT_SETTING, AzureStorageSettings.KEY_SETTING, + AzureStorageSettings.SAS_TOKEN_SETTING, AzureStorageSettings.ENDPOINT_SUFFIX_SETTING, AzureStorageSettings.TIMEOUT_SETTING, AzureStorageSettings.MAX_RETRIES_SETTING, diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java index fdd11e0f406cc..5fdf897f4c574 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java @@ -112,8 +112,8 @@ protected CloudBlobClient buildClient(AzureStorageSettings azureStorageSettings) return client; } - protected CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException { - final String connectionString = azureStorageSettings.buildConnectionString(); + private static CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException { + final String connectionString = azureStorageSettings.getConnectString(); return CloudStorageAccount.parse(connectionString).createCloudBlobClient(); } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java index f936235d300f9..aa0d1c499360d 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java @@ -21,6 +21,7 @@ import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.RetryPolicy; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.Tuple; @@ -57,6 +58,10 @@ public final class AzureStorageSettings { public static final AffixSetting KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key", key -> SecureSetting.secureString(key, null)); + /** Azure SAS token */ + public static final AffixSetting SAS_TOKEN_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "sas_token", + key -> SecureSetting.secureString(key, null)); + /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */ public static final Setting MAX_RETRIES_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries", @@ -118,7 +123,7 @@ public interface Storage { @Deprecated private final String name; private final String account; - private final String key; + private final String connectString; private final String endpointSuffix; private final TimeValue timeout; @Deprecated @@ -128,11 +133,11 @@ public interface Storage { private final LocationMode locationMode; // copy-constructor - private AzureStorageSettings(String name, String account, String key, String endpointSuffix, TimeValue timeout, boolean activeByDefault, - int maxRetries, Proxy proxy, LocationMode locationMode) { + private AzureStorageSettings(String name, String account, String connectString, String endpointSuffix, TimeValue timeout, + boolean activeByDefault, int maxRetries, Proxy proxy, LocationMode locationMode) { this.name = name; this.account = account; - this.key = key; + this.connectString = connectString; this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.activeByDefault = activeByDefault; @@ -145,7 +150,7 @@ private AzureStorageSettings(String name, String account, String key, String end public AzureStorageSettings(String name, String account, String key, TimeValue timeout, boolean activeByDefault, int maxRetries) { this.name = name; this.account = account; - this.key = key; + this.connectString = buildConnectString(account, key, null, null); this.endpointSuffix = null; this.timeout = timeout; this.activeByDefault = activeByDefault; @@ -154,11 +159,11 @@ public AzureStorageSettings(String name, String account, String key, TimeValue t this.locationMode = LocationMode.PRIMARY_ONLY; } - AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries, - Proxy.Type proxyType, String proxyHost, Integer proxyPort) { - this.name = null; + AzureStorageSettings(String name, String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries, + Proxy.Type proxyType, String proxyHost, Integer proxyPort) { + this.name = name; this.account = account; - this.key = key; + this.connectString = buildConnectString(account, key, sasToken, endpointSuffix); this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.activeByDefault = false; @@ -189,10 +194,6 @@ public String getName() { return name; } - public String getKey() { - return key; - } - public String getAccount() { return account; } @@ -218,13 +219,26 @@ public Proxy getProxy() { return proxy; } - public String buildConnectionString() { + public String getConnectString() { + return connectString; + } + + private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) { + final boolean hasSasToken = Strings.hasText(sasToken); + final boolean hasKey = Strings.hasText(key); + if (hasSasToken == false && hasKey == false) { + throw new SettingsException("Neither a secret key nor a shared access token was set."); + } + if (hasSasToken && hasKey) { + throw new SettingsException("Both a secret as well as a shared access token were set."); + } final StringBuilder connectionStringBuilder = new StringBuilder(); - connectionStringBuilder.append("DefaultEndpointsProtocol=https") - .append(";AccountName=") - .append(account) - .append(";AccountKey=") - .append(key); + connectionStringBuilder.append("DefaultEndpointsProtocol=https").append(";AccountName=").append(account); + if (hasKey) { + connectionStringBuilder.append(";AccountKey=").append(key); + } else { + connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken); + } if (Strings.hasText(endpointSuffix)) { connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix); } @@ -239,7 +253,6 @@ public LocationMode getLocationMode() { public String toString() { final StringBuilder sb = new StringBuilder("AzureStorageSettings{"); sb.append("account='").append(account).append('\''); - sb.append(", key='").append(key).append('\''); sb.append(", activeByDefault='").append(activeByDefault).append('\''); sb.append(", timeout=").append(timeout); sb.append(", endpointSuffix='").append(endpointSuffix).append('\''); @@ -309,8 +322,9 @@ static Map loadRegular(Settings settings) { /** Parse settings for a single client. */ static AzureStorageSettings getClientSettings(Settings settings, String clientName) { try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING); - SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) { - return new AzureStorageSettings(account.toString(), key.toString(), + SecureString key = getConfigValue(settings, clientName, KEY_SETTING); + SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)) { + return new AzureStorageSettings(null, account.toString(), key.toString(), sasToken.toString(), getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING), getValue(settings, clientName, TIMEOUT_SETTING), getValue(settings, clientName, MAX_RETRIES_SETTING), @@ -358,8 +372,8 @@ private static AzureStorageSettings getPrimary(List settin } else if (settings.size() == 1) { // the only storage settings belong (implicitly) to the default primary storage AzureStorageSettings storage = settings.get(0); - return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.getKey(), storage.getTimeout(), true, - storage.getMaxRetries()); + return new AzureStorageSettings(storage.getName(), storage.getAccount(), storage.connectString, null, storage.getTimeout(), + true, storage.getMaxRetries(), null, LocationMode.PRIMARY_ONLY); } else { AzureStorageSettings primary = null; for (AzureStorageSettings setting : settings) { @@ -398,8 +412,8 @@ public static Map overrideLocationMode(Map mapBuilder = new MapBuilder<>(); for (final Map.Entry entry : clientsSettings.entrySet()) { final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().name, entry.getValue().account, - entry.getValue().key, entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().activeByDefault, - entry.getValue().maxRetries, entry.getValue().proxy, locationMode); + entry.getValue().connectString, entry.getValue().endpointSuffix, entry.getValue().timeout, + entry.getValue().activeByDefault, entry.getValue().maxRetries, entry.getValue().proxy, locationMode); mapBuilder.put(entry.getKey(), azureSettings); } return mapBuilder.immutableMap(); diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java index 65eb2d7d7bbd5..11cedf7c6eb31 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSettingsParserTests.java @@ -49,11 +49,9 @@ public void testParseTwoSettingsExplicitDefault() { Tuple> tuple = AzureStorageSettings.loadLegacy(settings); assertThat(tuple.v1(), notNullValue()); assertThat(tuple.v1().getAccount(), is("myaccount1")); - assertThat(tuple.v1().getKey(), is("mykey1")); assertThat(tuple.v2().keySet(), hasSize(1)); assertThat(tuple.v2().get("azure2"), notNullValue()); assertThat(tuple.v2().get("azure2").getAccount(), is("myaccount2")); - assertThat(tuple.v2().get("azure2").getKey(), is("mykey2")); assertSettingDeprecationsAndWarnings(new Setting[]{ getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), getConcreteSetting(DEPRECATED_KEY_SETTING, "azure1"), @@ -72,7 +70,6 @@ public void testParseUniqueSettings() { Tuple> tuple = AzureStorageSettings.loadLegacy(settings); assertThat(tuple.v1(), notNullValue()); assertThat(tuple.v1().getAccount(), is("myaccount1")); - assertThat(tuple.v1().getKey(), is("mykey1")); assertThat(tuple.v2().keySet(), hasSize(0)); assertSettingDeprecationsAndWarnings(new Setting[]{ getConcreteSetting(DEPRECATED_ACCOUNT_SETTING, "azure1"), diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java index 7643ee85954ec..6878205e7521e 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java @@ -176,15 +176,21 @@ public void testReinitClientWrongSettings() throws IOException { secureSettings2.setString("azure.client.azure1.account", "myaccount1"); // missing key final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build(); + final MockSecureSettings secureSettings3 = new MockSecureSettings(); + secureSettings3.setString("azure.client.azure1.account", "myaccount3"); + secureSettings3.setString("azure.client.azure1.key", encodeKey("mykey33")); + secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33")); + final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) { final AzureStorageService azureStorageService = plugin.azureStoreService; final CloudBlobClient client11 = azureStorageService.client("azure1").v1(); assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net")); - plugin.reload(settings2); + final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2)); + assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set.")); + final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3)); + assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set.")); // existing client untouched assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net")); - final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure1")); - assertThat(e.getMessage(), is("Invalid azure client settings with name [azure1]")); } } From dc99f658eeb9a8c81b6ccaaf1e0051a8b3c508ef Mon Sep 17 00:00:00 2001 From: Armin Date: Mon, 1 Jul 2019 21:43:56 +0200 Subject: [PATCH 2/3] SAS token 3rd party tests --- .../qa/microsoft-azure-storage/build.gradle | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle index 812db5b438676..85a7cd42b560f 100644 --- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle +++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle @@ -64,8 +64,15 @@ processTestResources { integTestCluster { keystoreSetting 'azure.client.integration_test.account', azureAccount - keystoreSetting 'azure.client.integration_test.key', azureKey + if (azureKey != null && azureKey.isEmpty() == false) { + println "Using access key in external service tests." + keystoreSetting 'azure.client.integration_test.key', azureKey + } + if (azureSasToken != null && azureSasToken.isEmpty() == false) { + println "Using SAS token in external service tests." + keystoreSetting 'azure.client.integration_test.sas_token', azureSasToken + } if (useFixture) { dependsOn azureStorageFixture // Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used From 2af6e4cd5ab4dc714cefdc24f1f0f1ace0dda3f5 Mon Sep 17 00:00:00 2001 From: Armin Date: Mon, 1 Jul 2019 21:46:03 +0200 Subject: [PATCH 3/3] shorter diff --- .../repositories/azure/AzureStorageSettings.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java index aa0d1c499360d..bbcb9e7f583d3 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java @@ -159,9 +159,9 @@ public AzureStorageSettings(String name, String account, String key, TimeValue t this.locationMode = LocationMode.PRIMARY_ONLY; } - AzureStorageSettings(String name, String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries, - Proxy.Type proxyType, String proxyHost, Integer proxyPort) { - this.name = name; + private AzureStorageSettings(String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries, + Proxy.Type proxyType, String proxyHost, Integer proxyPort) { + this.name = null; this.account = account; this.connectString = buildConnectString(account, key, sasToken, endpointSuffix); this.endpointSuffix = endpointSuffix; @@ -324,7 +324,7 @@ static AzureStorageSettings getClientSettings(Settings settings, String clientNa try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING); SecureString key = getConfigValue(settings, clientName, KEY_SETTING); SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)) { - return new AzureStorageSettings(null, account.toString(), key.toString(), sasToken.toString(), + return new AzureStorageSettings(account.toString(), key.toString(), sasToken.toString(), getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING), getValue(settings, clientName, TIMEOUT_SETTING), getValue(settings, clientName, MAX_RETRIES_SETTING),