Add SAS Token support for authentification for azure repositories

This adds sas-token support for azure repositories based on elastic/elasticsearch#42982.

Author: mkleen

docs/appendices/release-notes/5.9.0.rst

@@ -86,5 +86,9 @@ None
 Administration and Operations
 -----------------------------
 
+- Added support for :ref:`Shared Access Signatures (SAS) tokens <sql-create-repo-azure-sas-token>`
+  as an alternative for authentication for :ref:`Azure repositories <sql-create-repo-azure>`.
+
 - Added ``id`` column to the :ref:`sys.snapshots <sys-snapshots>` table which
   exposes snapshot UUID.
+

docs/sql/statements/create-repository.rst

@@ -434,6 +434,25 @@ Parameters
       CrateDB masks this parameter. You cannot query the parameter value from
       the :ref:`sys.repositories <sys-repositories>` table.
 
+.. _sql-create-repo-azure-sas-token:
+
+**sas_token**
+  | *Type:*    ``text``
+
+  The Shared Access Signatures (SAS) token used for authentication for the Azure
+  Storage account. This can be used as an alternative to the Azure Storage
+  account secret key.
+
+  The SAS token must have read, write, list, and delete permissions for the
+  repository base path and all its contents. These permissions need to be
+  granted for the blob service and apply to resource types service, container,
+  and object.
+
+  .. NOTE::
+
+      CrateDB masks this parameter. You cannot query the parameter value from
+      the :ref:`sys.repositories <sys-repositories>` table.
+
 .. _sql-create-repo-azure-endpoint:
 
 **endpoint**

plugins/es-repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java

@@ -56,12 +56,8 @@ public AzureBlobStore(RepositoryMetadata metadata) {
     AzureBlobStore(RepositoryMetadata metadata, AzureStorageService service) {
         this.container = Repository.CONTAINER_SETTING.get(metadata.settings());
         this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
-
-        AzureStorageSettings repositorySettings = AzureStorageSettings
-            .getClientSettings(metadata.settings());
-        service.refreshSettings(repositorySettings);
-
         this.service = service;
+        this.service.refreshSettings(service.storageSettings);
     }
 
     @Override

plugins/es-repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

@@ -67,6 +67,9 @@ public class AzureRepository extends BlobStoreRepository {
     public static final String TYPE = "azure";
 
     public static final class Repository {
+
+        static final Setting<SecureString> SAS_TOKEN_SETTING = Setting.maskedString("sas_token");
+
         static final Setting<SecureString> ACCOUNT_SETTING = Setting.maskedString("account");
 
         static final Setting<SecureString> KEY_SETTING = Setting.maskedString("key");
@@ -147,24 +150,26 @@ public static final class Repository {
 
     public static List<Setting<?>> optionalSettings() {
         return List.of(Repository.CONTAINER_SETTING,
-                       Repository.BASE_PATH_SETTING,
-                       Repository.CHUNK_SIZE_SETTING,
-                       Repository.READONLY_SETTING,
-                       Repository.LOCATION_MODE_SETTING,
-                       COMPRESS_SETTING,
-                       // client specific repository settings
-                       Repository.MAX_RETRIES_SETTING,
-                       Repository.ENDPOINT_SETTING,
-                       Repository.SECONDARY_ENDPOINT_SETTING,
-                       Repository.ENDPOINT_SUFFIX_SETTING,
-                       Repository.TIMEOUT_SETTING,
-                       Repository.PROXY_TYPE_SETTING,
-                       Repository.PROXY_HOST_SETTING,
-                       Repository.PROXY_PORT_SETTING);
+            Repository.BASE_PATH_SETTING,
+            Repository.CHUNK_SIZE_SETTING,
+            Repository.READONLY_SETTING,
+            Repository.LOCATION_MODE_SETTING,
+            COMPRESS_SETTING,
+            // client specific repository settings
+            Repository.MAX_RETRIES_SETTING,
+            Repository.ENDPOINT_SETTING,
+            Repository.SECONDARY_ENDPOINT_SETTING,
+            Repository.ENDPOINT_SUFFIX_SETTING,
+            Repository.TIMEOUT_SETTING,
+            Repository.PROXY_TYPE_SETTING,
+            Repository.PROXY_HOST_SETTING,
+            Repository.PROXY_PORT_SETTING,
+            Repository.KEY_SETTING,
+            Repository.SAS_TOKEN_SETTING);
     }
 
     public static List<Setting<?>> mandatorySettings() {
-        return List.of(Repository.ACCOUNT_SETTING, Repository.KEY_SETTING);
+        return List.of(Repository.ACCOUNT_SETTING);
     }
 
     private final ByteSizeValue chunkSize;

plugins/es-repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -70,6 +70,6 @@ public Repository create(RepositoryMetadata metadata) throws Exception {
 
     @Override
     public List<Setting<?>> getSettings() {
-        return List.of(AzureRepository.Repository.ACCOUNT_SETTING, AzureRepository.Repository.KEY_SETTING);
+        return List.of(AzureRepository.Repository.ACCOUNT_SETTING);
     }
 }

plugins/es-repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java

@@ -121,7 +121,7 @@ protected CloudBlobClient buildClient(AzureStorageSettings azureStorageSettings)
     }
 
     protected CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
-        final String connectionString = azureStorageSettings.buildConnectionString();
+        final String connectionString = azureStorageSettings.getConnectString();
         return CloudStorageAccount.parse(connectionString).createCloudBlobClient();
     }
 

plugins/es-repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java

@@ -32,13 +32,14 @@
 
 import com.microsoft.azure.storage.LocationMode;
 
+import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.VisibleForTesting;
 import io.crate.common.unit.TimeValue;
 
 public final class AzureStorageSettings {
 
     private final String account;
-    private final String key;
+    private final String connectString;
     private final String endpoint;
     private final String secondaryEndpoint;
     private final String endpointSuffix;
@@ -49,7 +50,7 @@ public final class AzureStorageSettings {
 
     @VisibleForTesting
     AzureStorageSettings(String account,
-                         String key,
+                         String connectString,
                          String endpoint,
                          String secondaryEndpoint,
                          String endpointSuffix,
@@ -58,7 +59,7 @@ public final class AzureStorageSettings {
                          Proxy proxy,
                          LocationMode locationMode) {
         this.account = account;
-        this.key = key;
+        this.connectString = connectString;
         this.endpoint = endpoint;
         this.secondaryEndpoint = secondaryEndpoint;
         this.endpointSuffix = endpointSuffix;
@@ -69,16 +70,17 @@ public final class AzureStorageSettings {
     }
 
     private AzureStorageSettings(String account,
-                                 String key,
-                                 LocationMode locationMode,
-                                 String endpoint,
-                                 String secondaryEndpoint,
-                                 String endpointSuffix,
-                                 TimeValue timeout,
-                                 int maxRetries,
-                                 Proxy.Type proxyType,
-                                 String proxyHost,
-                                 Integer proxyPort) {
+                         String key,
+                         String sasToken,
+                         LocationMode locationMode,
+                         String endpoint,
+                         String secondaryEndpoint,
+                         String endpointSuffix,
+                         TimeValue timeout,
+                         int maxRetries,
+                         Proxy.Type proxyType,
+                         String proxyHost,
+                         Integer proxyPort) {
 
         final boolean hasEndpointSuffix = Strings.hasText(endpointSuffix);
         final boolean hasEndpoint = Strings.hasText(endpoint);
@@ -92,7 +94,7 @@ private AzureStorageSettings(String account,
         }
 
         this.account = account;
-        this.key = key;
+        this.connectString = buildConnectString(account, key, sasToken, endpoint, endpointSuffix, secondaryEndpoint);
         this.endpoint = endpoint;
         this.secondaryEndpoint = secondaryEndpoint;
         this.endpointSuffix = endpointSuffix;
@@ -131,18 +133,32 @@ public Proxy getProxy() {
         return proxy;
     }
 
+    public String getConnectString() {
+        return connectString;
+    }
+
     public LocationMode getLocationMode() {
         return locationMode;
     }
 
-    public String buildConnectionString() {
+    private String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpoint, String endpointSuffix, String secondaryEndpoint) {
+        boolean hasSasToken = Strings.hasText(sasToken);
+        boolean hasKey = Strings.hasText(key);
+        if (hasSasToken == false && hasKey == false) {
+            throw new SettingsException("Neither a secret key nor a shared access token was set.");
+        }
+        if (hasSasToken && hasKey) {
+            throw new SettingsException("Both a secret as well as a shared access token were set.");
+        }
         final StringBuilder connectionStringBuilder = new StringBuilder();
         connectionStringBuilder.append("DefaultEndpointsProtocol=https")
-                .append(";AccountName=")
-                .append(account)
-                .append(";AccountKey=")
-                .append(key);
-
+            .append(";AccountName=")
+            .append(account);
+        if (hasKey) {
+            connectionStringBuilder.append(";AccountKey=").append(key);
+        } else {
+            connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken);
+        }
         if (Strings.hasText(endpointSuffix)) {
             connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
         }
@@ -157,10 +173,12 @@ public String buildConnectionString() {
 
     static AzureStorageSettings getClientSettings(Settings settings) {
         try (SecureString account = getConfigValue(settings, AzureRepository.Repository.ACCOUNT_SETTING);
-             SecureString key = getConfigValue(settings, AzureRepository.Repository.KEY_SETTING)) {
+            SecureString key = getConfigValue(settings,AzureRepository.Repository.KEY_SETTING);
+            SecureString sasToken = getConfigValue(settings, AzureRepository.Repository.SAS_TOKEN_SETTING)) {
             return new AzureStorageSettings(
                 account.toString(),
                 key.toString(),
+                sasToken.toString(),
                 getConfigValue(settings, AzureRepository.Repository.LOCATION_MODE_SETTING),
                 getConfigValue(settings, AzureRepository.Repository.ENDPOINT_SETTING),
                 getConfigValue(settings, AzureRepository.Repository.SECONDARY_ENDPOINT_SETTING),
@@ -180,7 +198,7 @@ private static <T> T getConfigValue(Settings settings, Setting<T> clientSetting)
     static AzureStorageSettings copy(AzureStorageSettings settings) {
         return new AzureStorageSettings(
             settings.account,
-            settings.key,
+            settings.connectString,
             settings.endpoint,
             settings.secondaryEndpoint,
             settings.endpointSuffix,
@@ -193,7 +211,6 @@ static AzureStorageSettings copy(AzureStorageSettings settings) {
     @Override
     public String toString() {
         return "AzureStorageSettings{" + "account='" + account + '\'' +
-               ", key='" + key + '\'' +
                ", timeout=" + timeout +
                ", endpoint='" + endpoint + '\'' +
                ", secondaryEndpoint='" + secondaryEndpoint + '\'' +

plugins/es-repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryAnalyzerTest.java

@@ -74,14 +74,6 @@ private <S> S analyze(SQLExecutor e, String stmt) {
         }
     }
 
-    @Test
-    public void testCreateAzureRepoWithMissingMandatorySettings() {
-        assertThatThrownBy(
-            () -> analyze(e, "CREATE REPOSITORY foo TYPE azure WITH (account='test')"))
-            .isExactlyInstanceOf(IllegalArgumentException.class)
-            .hasMessage("The following required parameters are missing to create a repository of type \"azure\": [key]");
-    }
-
     @Test
     public void testCreateAzureRepositoryWithAllSettings() {
         PutRepositoryRequest request = analyze(

plugins/es-repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotIntegrationTest.java

@@ -150,6 +150,16 @@ public void create_azure_snapshot_and_restore_with_secondary_endpoint() {
         handler.blobs().keySet().forEach(x -> assertThat(x).doesNotEndWith("dat"));
     }
 
+    @Test
+    public void testCreateAzureRepoWithMissingMandatorySettings() {
+        Asserts.assertSQLError(() ->
+                execute("CREATE REPOSITORY r1 TYPE AZURE WITH (account = 'devstoreaccount1')"))
+            .hasPGError(INTERNAL_ERROR)
+            .hasHTTPError(INTERNAL_SERVER_ERROR, 5000)
+            .hasMessageContaining("[r1] Unable to verify the repository, [r1] is not accessible on master node: " +
+                "SettingsException 'Neither a secret key nor a shared access token was set.'");
+    }
+
     @Test
     public void test_invalid_settings_to_create_azure_repository() {
         Asserts.assertSQLError(() -> execute(