From ff37f150695572641cc1a1aa22faebdb2797ed13 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 6 Sep 2019 13:47:00 +0100 Subject: [PATCH 1/2] Add support for OneZoneInfrequentAccess storage The `repository-s3` plugin has supported a storage class of `onezone_ia` since the SDK upgrade in #30723, but we do not test or document this fact. This commit adds this storage class to the docs and adds a test to ensure that the documented storage classes are all accepted by S3 too. Fixes #30474 --- docs/plugins/repository-s3.asciidoc | 2 +- .../java/org/elasticsearch/repositories/s3/S3Repository.java | 2 +- .../org/elasticsearch/repositories/s3/S3BlobStoreTests.java | 4 +++- .../repositories/s3/S3RepositoryThirdPartyTests.java | 5 ++++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 454357a5d4b2e..43f84c42526a5 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -287,7 +287,7 @@ include::repository-shared-settings.asciidoc[] `storage_class`:: Sets the S3 storage class for objects stored in the snapshot repository. - Values may be `standard`, `reduced_redundancy`, `standard_ia` + Values may be `standard`, `reduced_redundancy`, `standard_ia`, `onezone_ia` and `intelligent_tiering`. Defaults to `standard`. Changing this setting on an existing repository only affects the storage class for newly created objects, resulting in a mixed usage of diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index a164900b635e7..af895758723f5 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -114,7 +114,7 @@ class S3Repository extends BlobStoreRepository { /** * Sets the S3 storage class type for the backup files. Values may be standard, reduced_redundancy, - * standard_ia and intelligent_tiering. Defaults to standard. + * standard_ia, onezone_ia and intelligent_tiering. Defaults to standard. */ static final Setting STORAGE_CLASS_SETTING = Setting.simpleString("storage_class"); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java index 10938504131ea..076ef4864a08d 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java @@ -74,9 +74,10 @@ public void testInitStorageClass() { assertThat(S3BlobStore.initStorageClass(null), equalTo(StorageClass.Standard)); assertThat(S3BlobStore.initStorageClass(""), equalTo(StorageClass.Standard)); - // it should accept [standard, standard_ia, reduced_redundancy, intelligent_tiering] + // it should accept [standard, standard_ia, onezone_ia, reduced_redundancy, intelligent_tiering] assertThat(S3BlobStore.initStorageClass("standard"), equalTo(StorageClass.Standard)); assertThat(S3BlobStore.initStorageClass("standard_ia"), equalTo(StorageClass.StandardInfrequentAccess)); + assertThat(S3BlobStore.initStorageClass("onezone_ia"), equalTo(StorageClass.OneZoneInfrequentAccess)); assertThat(S3BlobStore.initStorageClass("reduced_redundancy"), equalTo(StorageClass.ReducedRedundancy)); assertThat(S3BlobStore.initStorageClass("intelligent_tiering"), equalTo(StorageClass.IntelligentTiering)); } @@ -84,6 +85,7 @@ public void testInitStorageClass() { public void testCaseInsensitiveStorageClass() { assertThat(S3BlobStore.initStorageClass("sTandaRd"), equalTo(StorageClass.Standard)); assertThat(S3BlobStore.initStorageClass("sTandaRd_Ia"), equalTo(StorageClass.StandardInfrequentAccess)); + assertThat(S3BlobStore.initStorageClass("oNeZoNe_iA"), equalTo(StorageClass.OneZoneInfrequentAccess)); assertThat(S3BlobStore.initStorageClass("reduCED_redundancy"), equalTo(StorageClass.ReducedRedundancy)); assertThat(S3BlobStore.initStorageClass("intelLigeNt_tieriNG"), equalTo(StorageClass.IntelligentTiering)); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java index 28fd9c72f0868..595b4efdbfd00 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -63,7 +63,10 @@ protected void createRepository(String repoName) { .put("base_path", System.getProperty("test.s3.base", "testpath")); final String endpoint = System.getProperty("test.s3.endpoint"); if (endpoint != null) { - settings = settings.put("endpoint", endpoint); + settings.put("endpoint", endpoint); + } + if (randomBoolean()) { + settings.put("storage_class", randomFrom("standard", "reduced_redundancy", "standard_ia", "onezone_ia", "intelligent_tiering")); } AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo") .setType("s3") From c1242c02d52cf72f81911d2a7d44c9e79eeb62af Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 6 Sep 2019 14:07:48 +0100 Subject: [PATCH 2/2] Only test non-default storage classes when running against S3 --- .../repositories/s3/S3RepositoryThirdPartyTests.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java index 595b4efdbfd00..1f0fcc2a97bf0 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -64,9 +64,14 @@ protected void createRepository(String repoName) { final String endpoint = System.getProperty("test.s3.endpoint"); if (endpoint != null) { settings.put("endpoint", endpoint); - } - if (randomBoolean()) { - settings.put("storage_class", randomFrom("standard", "reduced_redundancy", "standard_ia", "onezone_ia", "intelligent_tiering")); + } else { + // only test different storage classes when running against the default endpoint, i.e. a genuine S3 service + if (randomBoolean()) { + final String storageClass + = randomFrom("standard", "reduced_redundancy", "standard_ia", "onezone_ia", "intelligent_tiering"); + logger.info("--> using storage_class [{}]", storageClass); + settings.put("storage_class", storageClass); + } } AcknowledgedResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo") .setType("s3")