From 713698a4a643d54be4235a3f3db410d9c20e348e Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 8 May 2019 21:23:31 +0200 Subject: [PATCH 01/10] Provide an Option to Use Path-Style-Access with S3 Repo * As discussed, added the option to use path style access back again and deprecated it. * Defaulted to `false` * Added warning to docs * Closes #41816 --- docs/plugins/repository-s3.asciidoc | 15 +++++++++++---- .../repositories/s3/S3ClientSettings.java | 18 +++++++++++++++--- .../repositories/s3/S3RepositoryPlugin.java | 1 + .../repositories/s3/S3Service.java | 6 ++++-- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index e2e489bb93eea..e054e56cf2275 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -145,6 +145,17 @@ settings belong in the `elasticsearch.yml` file. Whether retries should be throttled (i.e. should back off). Must be `true` or `false`. Defaults to `true`. +`path_style_access`:: + + Whether to use the path style access pattern. Defaults to `false`. + +NOTE: This setting is deprecated and only intended as a stop-gap solution to provide + compatibility with alternative S3 implementations that do not provide compatibility with the domain style access pattern. + AWS S3 will stop supporting the path style access pattern from + https://forums.aws.amazon.com/ann.jspa?annID=6776[September 30th, 2020]. Releases of Elasticsearch will likely stop supporting + this setting before that data. Any setups using the path style access pattern should be switched to the domain style access pattern + as soon as possible to ensure continued compatibility with the S3 repository plugin. + [float] [[repository-s3-compatible-services]] ===== S3-compatible services @@ -381,10 +392,6 @@ bucket, in this example, named "foo". The bucket needs to exist to register a repository for snapshots. If you did not create the bucket then the repository registration will fail. -Note: Starting in version 7.0, all bucket operations are using the path style -access pattern. In previous versions the decision to use virtual hosted style or -path style access was made by the AWS Java SDK. - [[repository-s3-aws-vpc]] [float] ==== AWS VPC Bandwidth Settings diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index ea45fbaf93dd3..fb6bf616a69bf 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -95,6 +95,10 @@ final class S3ClientSettings { static final Setting.AffixSetting USE_THROTTLE_RETRIES_SETTING = Setting.affixKeySetting(PREFIX, "use_throttle_retries", key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); + /** Whether the s3 client should use path style access. */ + static final Setting.AffixSetting USE_PATH_STYLE_ACCESS = Setting.affixKeySetting(PREFIX, "path_style_access", + key -> Setting.boolSetting(key, false, Property.NodeScope, Property.Deprecated)); + /** Credentials to authenticate with s3. */ final S3BasicCredentials credentials; @@ -127,9 +131,13 @@ final class S3ClientSettings { /** Whether the s3 client should use an exponential backoff retry policy. */ final boolean throttleRetries; + /** Whether the s3 client should use path style access. */ + final boolean pathStyleAccess; + private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, - int readTimeoutMillis, int maxRetries, boolean throttleRetries) { + int readTimeoutMillis, int maxRetries, boolean throttleRetries, + boolean pathStyleAccess) { this.credentials = credentials; this.endpoint = endpoint; this.protocol = protocol; @@ -140,6 +148,7 @@ private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protoc this.readTimeoutMillis = readTimeoutMillis; this.maxRetries = maxRetries; this.throttleRetries = throttleRetries; + this.pathStyleAccess = pathStyleAccess; } /** @@ -162,6 +171,7 @@ S3ClientSettings refine(RepositoryMetaData metadata) { getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis()); final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); + final boolean usePathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); final S3BasicCredentials newCredentials; if (checkDeprecatedCredentials(repoSettings)) { newCredentials = loadDeprecatedCredentials(repoSettings); @@ -183,7 +193,8 @@ S3ClientSettings refine(RepositoryMetaData metadata) { proxyPassword, newReadTimeoutMillis, newMaxRetries, - newThrottleRetries + newThrottleRetries, + usePathStyleAccess ); } @@ -270,7 +281,8 @@ static S3ClientSettings getClientSettings(final Settings settings, final String proxyPassword.toString(), Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), getConfigValue(settings, clientName, MAX_RETRIES_SETTING), - getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING) + getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), + getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS) ); } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index bb044771e6085..118197902f600 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -105,6 +105,7 @@ public List> getSettings() { S3ClientSettings.READ_TIMEOUT_SETTING, S3ClientSettings.MAX_RETRIES_SETTING, S3ClientSettings.USE_THROTTLE_RETRIES_SETTING, + S3ClientSettings.USE_PATH_STYLE_ACCESS, S3Repository.ACCESS_KEY_SETTING, S3Repository.SECRET_KEY_SETTING); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 89afc7eefeee3..79db2c4fa0392 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -153,8 +153,10 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { // // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too) // so this change removes that usage of a deprecated API. - builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)) - .enablePathStyleAccess(); + builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); + if (clientSettings.pathStyleAccess) { + builder.enablePathStyleAccess(); + } return builder.build(); } From 13e33114d9f5b303cd03921741b7d169fb569baf Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 8 May 2019 21:35:43 +0200 Subject: [PATCH 02/10] add breaking changes entry --- .../migration/migrate_8_0/snapshots.asciidoc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc index 791e5b28da057..bb034d0942ec3 100644 --- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc +++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc @@ -28,3 +28,15 @@ This change will affect both newly created repositories and existing repositorie explicitly specified. For more information on the compress option, see <> + +[float] +==== The S3 repository plugin uses the DNS style access pattern by default + +Starting in version 7.2 using the path style access pattern with the S3 repository is deprecated. +Previously the S3 repository plugin was exclusively using the path style access pattern so this is a breaking +change for deployments that do not also allow for the DNS style access pattern. For short-term compatibility with these deployments users +must configure the S3 client setting `path_style_access` to `true` to retain the previous behaviour but should be advised that +the option to enable path style access will be removed in future versions of Elasticsearch. + +This breaking change was made necessary by https://forums.aws.amazon.com/ann.jspa?annID=6776[AWS's announcement] to no longer support +the path-style API past September 30th, 2020. From 5c85170b74f46c51a6d1e0e68dc0a20a21cc460c Mon Sep 17 00:00:00 2001 From: Armin Date: Mon, 3 Jun 2019 15:57:04 +0200 Subject: [PATCH 03/10] default to SDK defaults --- docs/plugins/repository-s3.asciidoc | 11 ++++--- .../migration/migrate_8_0/snapshots.asciidoc | 12 +++---- .../repositories/s3/S3ClientSettings.java | 31 ++++++++++++++++--- .../repositories/s3/S3Service.java | 7 ++++- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index e054e56cf2275..a977f5c5698fc 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -147,14 +147,15 @@ settings belong in the `elasticsearch.yml` file. `path_style_access`:: - Whether to use the path style access pattern. Defaults to `false`. + Whether to use the path style access pattern. If `true`, the path style access pattern will be used if false, DNS style access will + be used. Defaults to letting the AWS S3 SDK decide the right access pattern dynamically. NOTE: This setting is deprecated and only intended as a stop-gap solution to provide compatibility with alternative S3 implementations that do not provide compatibility with the domain style access pattern. - AWS S3 will stop supporting the path style access pattern from - https://forums.aws.amazon.com/ann.jspa?annID=6776[September 30th, 2020]. Releases of Elasticsearch will likely stop supporting - this setting before that data. Any setups using the path style access pattern should be switched to the domain style access pattern - as soon as possible to ensure continued compatibility with the S3 repository plugin. + AWS S3 will stop supporting the path style access pattern for new buckets from + https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story[September 30th, 2020]. + Any setups using the path style access pattern should be switched to the domain style access pattern as soon as possible to ensure + continued compatibility with the S3 repository plugin. [float] [[repository-s3-compatible-services]] diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc index bb034d0942ec3..257a29274e580 100644 --- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc +++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc @@ -32,11 +32,11 @@ For more information on the compress option, see <> [float] ==== The S3 repository plugin uses the DNS style access pattern by default -Starting in version 7.2 using the path style access pattern with the S3 repository is deprecated. -Previously the S3 repository plugin was exclusively using the path style access pattern so this is a breaking +Starting in version 7.3 using the path style access pattern with the S3 repository is deprecated. +Previously the S3 repository plugin was exclusively using the path style access pattern. This is a breaking change for deployments that do not also allow for the DNS style access pattern. For short-term compatibility with these deployments users -must configure the S3 client setting `path_style_access` to `true` to retain the previous behaviour but should be advised that -the option to enable path style access will be removed in future versions of Elasticsearch. +must configure the S3 client setting `path_style_access` to `true` to retain the previous behaviour. -This breaking change was made necessary by https://forums.aws.amazon.com/ann.jspa?annID=6776[AWS's announcement] to no longer support -the path-style API past September 30th, 2020. +This breaking change was made necessary by +https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story[AWS's announcement] to no longer support +the path-style API past September 30th, 2020 for newly created buckets. diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index fb6bf616a69bf..cccd69eca6349 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -21,6 +21,7 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; +import com.amazonaws.services.s3.AmazonS3Builder; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; @@ -96,8 +97,10 @@ final class S3ClientSettings { key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); /** Whether the s3 client should use path style access. */ - static final Setting.AffixSetting USE_PATH_STYLE_ACCESS = Setting.affixKeySetting(PREFIX, "path_style_access", - key -> Setting.boolSetting(key, false, Property.NodeScope, Property.Deprecated)); + static final Setting.AffixSetting USE_PATH_STYLE_ACCESS = Setting.affixKeySetting( + PREFIX, "path_style_access", + key -> new Setting<>(key, s -> "default", s -> Boolean.parseBoolean(s) ? PathStyleAccess.ENABLED : PathStyleAccess.DISABLED, + Property.NodeScope, Property.Deprecated)); /** Credentials to authenticate with s3. */ final S3BasicCredentials credentials; @@ -132,12 +135,12 @@ final class S3ClientSettings { final boolean throttleRetries; /** Whether the s3 client should use path style access. */ - final boolean pathStyleAccess; + final PathStyleAccess pathStyleAccess; private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries, - boolean pathStyleAccess) { + PathStyleAccess pathStyleAccess) { this.credentials = credentials; this.endpoint = endpoint; this.protocol = protocol; @@ -171,7 +174,7 @@ S3ClientSettings refine(RepositoryMetaData metadata) { getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis()); final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); - final boolean usePathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); + final PathStyleAccess usePathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); final S3BasicCredentials newCredentials; if (checkDeprecatedCredentials(repoSettings)) { newCredentials = loadDeprecatedCredentials(repoSettings); @@ -326,4 +329,22 @@ private static T getRepoSettingOrDefault(Setting.AffixSetting setting, Se } return defaultValue; } + + /** + * Settings for path style access behavior. + */ + enum PathStyleAccess { + /** + * Let SDK decide whether to use path style access. + */ + DEFAULT, + /** + * Force use of path style access, i.e set {@link AmazonS3Builder#setPathStyleAccessEnabled(Boolean)} to {@code true}. + */ + ENABLED, + /** + * Don't use path style access, i.e set {@link AmazonS3Builder#setPathStyleAccessEnabled(Boolean)} to {@code false}. + */ + DISABLED + } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 79db2c4fa0392..3b2c4dabf4f7c 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -154,8 +154,13 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too) // so this change removes that usage of a deprecated API. builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); - if (clientSettings.pathStyleAccess) { + if (clientSettings.pathStyleAccess == S3ClientSettings.PathStyleAccess.ENABLED) { builder.enablePathStyleAccess(); + } else if (clientSettings.pathStyleAccess == S3ClientSettings.PathStyleAccess.DISABLED) { + builder.setPathStyleAccessEnabled(false); + } else { + assert clientSettings.pathStyleAccess == S3ClientSettings.PathStyleAccess.DEFAULT; + assert builder.isPathStyleAccessEnabled() == null; } return builder.build(); From 22ecda1744fcf6eec6b68b6a5655f57e8067368a Mon Sep 17 00:00:00 2001 From: Armin Date: Tue, 4 Jun 2019 13:37:13 +0200 Subject: [PATCH 04/10] CR: Adjust docs --- docs/plugins/repository-s3.asciidoc | 5 +++-- docs/reference/migration/migrate_8_0/snapshots.asciidoc | 2 +- .../org/elasticsearch/repositories/s3/S3ClientSettings.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index a977f5c5698fc..3c405b7e44dfe 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -147,8 +147,9 @@ settings belong in the `elasticsearch.yml` file. `path_style_access`:: - Whether to use the path style access pattern. If `true`, the path style access pattern will be used if false, DNS style access will - be used. Defaults to letting the AWS S3 SDK decide the right access pattern dynamically. + Whether to use the path style access pattern. If `true`, the path style access pattern will be used. If set to`false`, DNS style access will + be used. Defaults to letting the AWS Java SDK decide whether to use the path style access pattern dynamically + (See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS documentation] for details). NOTE: This setting is deprecated and only intended as a stop-gap solution to provide compatibility with alternative S3 implementations that do not provide compatibility with the domain style access pattern. diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc index 257a29274e580..179766a258f3b 100644 --- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc +++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc @@ -33,7 +33,7 @@ For more information on the compress option, see <> ==== The S3 repository plugin uses the DNS style access pattern by default Starting in version 7.3 using the path style access pattern with the S3 repository is deprecated. -Previously the S3 repository plugin was exclusively using the path style access pattern. This is a breaking +In versions 7.0, 7.1, and 7.2 the S3 repository plugin was exclusively using the path style access pattern. This is a breaking change for deployments that do not also allow for the DNS style access pattern. For short-term compatibility with these deployments users must configure the S3 client setting `path_style_access` to `true` to retain the previous behaviour. diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index cccd69eca6349..0ada6c378e6ff 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -100,7 +100,7 @@ final class S3ClientSettings { static final Setting.AffixSetting USE_PATH_STYLE_ACCESS = Setting.affixKeySetting( PREFIX, "path_style_access", key -> new Setting<>(key, s -> "default", s -> Boolean.parseBoolean(s) ? PathStyleAccess.ENABLED : PathStyleAccess.DISABLED, - Property.NodeScope, Property.Deprecated)); + Property.NodeScope)); /** Credentials to authenticate with s3. */ final S3BasicCredentials credentials; From 3110d3cc9950c4bb53e728fbd486fe6a93c47ef8 Mon Sep 17 00:00:00 2001 From: Armin Date: Tue, 11 Jun 2019 17:26:06 +0200 Subject: [PATCH 05/10] boolean setting --- .../repositories/s3/S3ClientSettings.java | 31 +++---------------- .../repositories/s3/S3Service.java | 8 +---- .../s3/S3ClientSettingsTests.java | 7 +++++ 3 files changed, 13 insertions(+), 33 deletions(-) diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index 0ada6c378e6ff..ae2bd2e905bf6 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -21,7 +21,6 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; -import com.amazonaws.services.s3.AmazonS3Builder; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; @@ -97,10 +96,8 @@ final class S3ClientSettings { key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); /** Whether the s3 client should use path style access. */ - static final Setting.AffixSetting USE_PATH_STYLE_ACCESS = Setting.affixKeySetting( - PREFIX, "path_style_access", - key -> new Setting<>(key, s -> "default", s -> Boolean.parseBoolean(s) ? PathStyleAccess.ENABLED : PathStyleAccess.DISABLED, - Property.NodeScope)); + static final Setting.AffixSetting USE_PATH_STYLE_ACCESS = Setting.affixKeySetting(PREFIX, "path_style_access", + key -> Setting.boolSetting(key, false, Property.NodeScope)); /** Credentials to authenticate with s3. */ final S3BasicCredentials credentials; @@ -135,12 +132,12 @@ final class S3ClientSettings { final boolean throttleRetries; /** Whether the s3 client should use path style access. */ - final PathStyleAccess pathStyleAccess; + final boolean pathStyleAccess; private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries, - PathStyleAccess pathStyleAccess) { + boolean pathStyleAccess) { this.credentials = credentials; this.endpoint = endpoint; this.protocol = protocol; @@ -174,7 +171,7 @@ S3ClientSettings refine(RepositoryMetaData metadata) { getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis()); final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); - final PathStyleAccess usePathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); + final boolean usePathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); final S3BasicCredentials newCredentials; if (checkDeprecatedCredentials(repoSettings)) { newCredentials = loadDeprecatedCredentials(repoSettings); @@ -329,22 +326,4 @@ private static T getRepoSettingOrDefault(Setting.AffixSetting setting, Se } return defaultValue; } - - /** - * Settings for path style access behavior. - */ - enum PathStyleAccess { - /** - * Let SDK decide whether to use path style access. - */ - DEFAULT, - /** - * Force use of path style access, i.e set {@link AmazonS3Builder#setPathStyleAccessEnabled(Boolean)} to {@code true}. - */ - ENABLED, - /** - * Don't use path style access, i.e set {@link AmazonS3Builder#setPathStyleAccessEnabled(Boolean)} to {@code false}. - */ - DISABLED - } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 3b2c4dabf4f7c..3b232354ddfea 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -154,15 +154,9 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too) // so this change removes that usage of a deprecated API. builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); - if (clientSettings.pathStyleAccess == S3ClientSettings.PathStyleAccess.ENABLED) { + if (clientSettings.pathStyleAccess) { builder.enablePathStyleAccess(); - } else if (clientSettings.pathStyleAccess == S3ClientSettings.PathStyleAccess.DISABLED) { - builder.setPathStyleAccessEnabled(false); - } else { - assert clientSettings.pathStyleAccess == S3ClientSettings.PathStyleAccess.DEFAULT; - assert builder.isPathStyleAccessEnabled() == null; } - return builder.build(); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index 53740672df329..312d9649aa375 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -144,4 +144,11 @@ public void testRefineWithRepoSettings() { assertThat(credentials.getSessionToken(), is("session_token")); } } + + public void testPathStyleAccessCanBeSet() { + final Map settings = S3ClientSettings.load( + Settings.builder().put("s3.client.other.path_style_access", true).build()); + assertThat(settings.get("default").pathStyleAccess, is(false)); + assertThat(settings.get("other").pathStyleAccess, is(true)); + } } From 42fa7e4690990f01f36eca9d20b0c64076f603c5 Mon Sep 17 00:00:00 2001 From: Armin Date: Tue, 11 Jun 2019 17:43:16 +0200 Subject: [PATCH 06/10] CR: tone down language :) --- docs/plugins/repository-s3.asciidoc | 14 ++++++-------- .../migration/migrate_8_0/snapshots.asciidoc | 6 +++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 3c405b7e44dfe..4f5fd428577ed 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -147,16 +147,14 @@ settings belong in the `elasticsearch.yml` file. `path_style_access`:: - Whether to use the path style access pattern. If `true`, the path style access pattern will be used. If set to`false`, DNS style access will - be used. Defaults to letting the AWS Java SDK decide whether to use the path style access pattern dynamically + Whether to use the path style access pattern. If `true`, the path style access pattern will be used. If set to`false`, + the AWS Java SDK decide whether to use the path style access pattern dynamically (See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS documentation] for details). + Defaults to `false`. -NOTE: This setting is deprecated and only intended as a stop-gap solution to provide - compatibility with alternative S3 implementations that do not provide compatibility with the domain style access pattern. - AWS S3 will stop supporting the path style access pattern for new buckets from - https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story[September 30th, 2020]. - Any setups using the path style access pattern should be switched to the domain style access pattern as soon as possible to ensure - continued compatibility with the S3 repository plugin. +NOTE: In versions `7.0` and `7.1`, all bucket operations were using the path style access pattern in every situation without an option to +disable it. If your deployment requires path style access to be used you might need to manually configure this setting to `true` when +upgrading. [float] [[repository-s3-compatible-services]] diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc index 179766a258f3b..cafc3acd03272 100644 --- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc +++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc @@ -34,9 +34,9 @@ For more information on the compress option, see <> Starting in version 7.3 using the path style access pattern with the S3 repository is deprecated. In versions 7.0, 7.1, and 7.2 the S3 repository plugin was exclusively using the path style access pattern. This is a breaking -change for deployments that do not also allow for the DNS style access pattern. For short-term compatibility with these deployments users -must configure the S3 client setting `path_style_access` to `true` to retain the previous behaviour. - +change for deployments that do not also allow for the DNS style but are recognized as supporting DNS style access by the AWS SDK. +If your deployment does only support path style access and is affected by this change you must configure the S3 client setting +`path_style_access` to `true` to return to the behaviour of always using path style access. This breaking change was made necessary by https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story[AWS's announcement] to no longer support the path-style API past September 30th, 2020 for newly created buckets. From af3f0a16bb40984471a4e4c8140df3494a6a6371 Mon Sep 17 00:00:00 2001 From: Armin Date: Tue, 11 Jun 2019 17:51:14 +0200 Subject: [PATCH 07/10] tone it down some more --- docs/reference/migration/migrate_8_0/snapshots.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc index cafc3acd03272..2eeb9c81b5495 100644 --- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc +++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc @@ -32,7 +32,7 @@ For more information on the compress option, see <> [float] ==== The S3 repository plugin uses the DNS style access pattern by default -Starting in version 7.3 using the path style access pattern with the S3 repository is deprecated. +Starting in version 7.3 using the path style access pattern with the S3 repository is not enabled by default. In versions 7.0, 7.1, and 7.2 the S3 repository plugin was exclusively using the path style access pattern. This is a breaking change for deployments that do not also allow for the DNS style but are recognized as supporting DNS style access by the AWS SDK. If your deployment does only support path style access and is affected by this change you must configure the S3 client setting From c13bf0cc6ff68a4dff5787665ac34add701a6c14 Mon Sep 17 00:00:00 2001 From: Armin Date: Mon, 17 Jun 2019 11:18:39 +0200 Subject: [PATCH 08/10] CR: fix docs --- docs/plugins/repository-s3.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 4f5fd428577ed..7675888f1a92a 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -147,12 +147,12 @@ settings belong in the `elasticsearch.yml` file. `path_style_access`:: - Whether to use the path style access pattern. If `true`, the path style access pattern will be used. If set to`false`, - the AWS Java SDK decide whether to use the path style access pattern dynamically + Whether to force the use of the path style access pattern. If `true`, the path style access pattern will be used. + If left to`false`, path style access will be automatically determined by the AWS Java SDK (See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS documentation] for details). Defaults to `false`. -NOTE: In versions `7.0` and `7.1`, all bucket operations were using the path style access pattern in every situation without an option to +NOTE: In versions `7.0`, 7.1`, and `7.2`, all bucket operations were using the path style access pattern in every situation without an option to disable it. If your deployment requires path style access to be used you might need to manually configure this setting to `true` when upgrading. From f76a14ea33dea00aa89a0df3422c2220b2af5540 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 4 Jul 2019 13:13:06 +0100 Subject: [PATCH 09/10] Docs fixes post 7.3 FF --- docs/plugins/repository-s3.asciidoc | 19 +++++++++++-------- .../migration/migrate_8_0/snapshots.asciidoc | 19 +++++++++++-------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 7675888f1a92a..78f53b587f7e2 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -147,14 +147,17 @@ settings belong in the `elasticsearch.yml` file. `path_style_access`:: - Whether to force the use of the path style access pattern. If `true`, the path style access pattern will be used. - If left to`false`, path style access will be automatically determined by the AWS Java SDK - (See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS documentation] for details). - Defaults to `false`. - -NOTE: In versions `7.0`, 7.1`, and `7.2`, all bucket operations were using the path style access pattern in every situation without an option to -disable it. If your deployment requires path style access to be used you might need to manually configure this setting to `true` when -upgrading. + Whether to force the use of the path style access pattern. If `true`, the + path style access pattern will be used. If `false`, path style access will be + automatically determined by the AWS Java SDK (See + https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS + documentation] for details). Defaults to `false`. + +[[repository-s3-path-style-deprecation]] +NOTE: In versions `7.0`, `7.1`, `7.2` and `7.3` all bucket operations used the +https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[now-deprecated] +path style access pattern. If your deployment requires the path style access +pattern then you should set this setting to `true` when upgrading. [float] [[repository-s3-compatible-services]] diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc index 2eeb9c81b5495..2f10845132664 100644 --- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc +++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc @@ -32,11 +32,14 @@ For more information on the compress option, see <> [float] ==== The S3 repository plugin uses the DNS style access pattern by default -Starting in version 7.3 using the path style access pattern with the S3 repository is not enabled by default. -In versions 7.0, 7.1, and 7.2 the S3 repository plugin was exclusively using the path style access pattern. This is a breaking -change for deployments that do not also allow for the DNS style but are recognized as supporting DNS style access by the AWS SDK. -If your deployment does only support path style access and is affected by this change you must configure the S3 client setting -`path_style_access` to `true` to return to the behaviour of always using path style access. -This breaking change was made necessary by -https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story[AWS's announcement] to no longer support -the path-style API past September 30th, 2020 for newly created buckets. +Starting in version 7.4 the `repository-s3` plugin does not use the +now-deprecated path-style access pattern by default. In versions 7.0, 7.1, 7.2 +and 7.3 the `repository-s3` plugin always used the path-style access pattern. +This is a breaking change for deployments that only support path-style access +but which are recognized as supporting DNS-style access by the AWS SDK. If your +deployment only supports path-style access and is affected by this change then +you must configure the S3 client setting `path_style_access` to `true`. This +breaking change was made necessary by +https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[AWS's +announcement] that the path-style access pattern is deprecated and will be +unsupported on buckets created after September 30th 2020. From 5ba5ac413698c70cac658f817c70ab656c215116 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 4 Jul 2019 13:15:09 +0100 Subject: [PATCH 10/10] Wording tweak --- docs/plugins/repository-s3.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 78f53b587f7e2..48b03f1abc156 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -148,8 +148,8 @@ settings belong in the `elasticsearch.yml` file. `path_style_access`:: Whether to force the use of the path style access pattern. If `true`, the - path style access pattern will be used. If `false`, path style access will be - automatically determined by the AWS Java SDK (See + path style access pattern will be used. If `false`, the access pattern will + be automatically determined by the AWS Java SDK (See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS documentation] for details). Defaults to `false`.