From 1e32c08311cd72e5213232204edc4023dfe67cef Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 30 Mar 2018 12:10:27 +0200 Subject: [PATCH 1/3] Use ESBlobStoreRepositoryIntegTestCase to test the repository-s3 plugin The test framework contains a base class for testing blob store repository implementations, but the S3 plugin does not use it. This commit adds the S3BlobStoreRepositoryTests class that extends the base testing class for S3. It also cleans up the S3BlobStoreTests and S3BlobStoreContainerTests so that they are now based on pure mock S3 clients. It also removes some usage of socket servers that emulate socket connections in unit tests. It was added to trigger security exceptions, but this won't be needed anymore once #29296 will be merged. closes #16472 --- .../repositories/s3/S3Repository.java | 3 +- .../repositories/s3/MockAmazonS3.java | 338 ++++++++---------- .../s3/S3BlobStoreContainerTests.java | 56 +-- .../s3/S3BlobStoreRepositoryTests.java | 100 ++++++ .../repositories/s3/S3BlobStoreTests.java | 75 ++-- 5 files changed, 309 insertions(+), 263 deletions(-) create mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 51bb6f2024cd4..09d9782aa91f8 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -151,8 +151,7 @@ class S3Repository extends BlobStoreRepository { /** * Constructs an s3 backed repository */ - S3Repository(RepositoryMetaData metadata, Settings settings, - NamedXContentRegistry namedXContentRegistry, AwsS3Service s3Service) throws IOException { + S3Repository(RepositoryMetaData metadata, Settings settings, NamedXContentRegistry namedXContentRegistry, AwsS3Service s3Service) { super(metadata, settings, namedXContentRegistry); String bucket = BUCKET_SETTING.get(metadata.settings()); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java index a090fdd5281fd..fe845398f8dce 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java @@ -19,215 +19,177 @@ package org.elasticsearch.repositories.s3; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AbstractAmazonS3; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.elasticsearch.common.io.Streams; -import java.io.IOException; -import java.io.InputStream; -import java.io.UncheckedIOException; -import java.net.InetAddress; -import java.net.Socket; -import java.util.ArrayList; -import java.util.List; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static org.junit.Assert.assertTrue; +import static java.util.Collections.emptyMap; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class MockAmazonS3 extends AbstractAmazonS3 { - private final int mockSocketPort; + static final Map> BUCKETS = new ConcurrentHashMap<>(); - private Map blobs = new ConcurrentHashMap<>(); - - // in ESBlobStoreContainerTestCase.java, the maximum - // length of the input data is 100 bytes - private byte[] byteCounter = new byte[100]; - - - MockAmazonS3(int mockSocketPort) { - this.mockSocketPort = mockSocketPort; + private MockAmazonS3() { } - // Simulate a socket connection to check that SocketAccess.doPrivileged() is used correctly. - // Any method of AmazonS3 might potentially open a socket to the S3 service. Firstly, a call - // to any method of AmazonS3 has to be wrapped by SocketAccess.doPrivileged(). - // Secondly, each method on the stack from doPrivileged to opening the socket has to be - // located in a jar that is provided by the plugin. - // Thirdly, a SocketPermission has to be configured in plugin-security.policy. - // By opening a socket in each method of MockAmazonS3 it is ensured that in production AmazonS3 - // is able to to open a socket to the S3 Service without causing a SecurityException - private void simulateS3SocketConnection() { - try (Socket socket = new Socket(InetAddress.getByName("127.0.0.1"), mockSocketPort)) { - assertTrue(socket.isConnected()); // NOOP to keep static analysis happy - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - - @Override - public boolean doesBucketExist(String bucket) { - return true; - } - - @Override - public boolean doesObjectExist(String bucketName, String objectName) throws AmazonServiceException, SdkClientException { - simulateS3SocketConnection(); - return blobs.containsKey(objectName); - } - - @Override - public ObjectMetadata getObjectMetadata( - GetObjectMetadataRequest getObjectMetadataRequest) - throws AmazonClientException, AmazonServiceException { - simulateS3SocketConnection(); - String blobName = getObjectMetadataRequest.getKey(); - - if (!blobs.containsKey(blobName)) { - throw new AmazonS3Exception("[" + blobName + "] does not exist."); - } - - return new ObjectMetadata(); // nothing is done with it - } - - @Override - public PutObjectResult putObject(PutObjectRequest putObjectRequest) - throws AmazonClientException, AmazonServiceException { - simulateS3SocketConnection(); - String blobName = putObjectRequest.getKey(); - - if (blobs.containsKey(blobName)) { - throw new AmazonS3Exception("[" + blobName + "] already exists."); - } - - blobs.put(blobName, putObjectRequest.getInputStream()); - return new PutObjectResult(); - } - - @Override - public S3Object getObject(GetObjectRequest getObjectRequest) - throws AmazonClientException, AmazonServiceException { - simulateS3SocketConnection(); - // in ESBlobStoreContainerTestCase.java, the prefix is empty, - // so the key and blobName are equivalent to each other - String blobName = getObjectRequest.getKey(); - - if (!blobs.containsKey(blobName)) { - throw new AmazonS3Exception("[" + blobName + "] does not exist."); - } - - // the HTTP request attribute is irrelevant for reading - S3ObjectInputStream stream = new S3ObjectInputStream( - blobs.get(blobName), null, false); - S3Object s3Object = new S3Object(); - s3Object.setObjectContent(stream); - return s3Object; - } - - @Override - public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) - throws AmazonClientException, AmazonServiceException { - simulateS3SocketConnection(); - MockObjectListing list = new MockObjectListing(); - list.setTruncated(false); - - String blobName; - String prefix = listObjectsRequest.getPrefix(); - - ArrayList mockObjectSummaries = new ArrayList<>(); - - for (Map.Entry blob : blobs.entrySet()) { - blobName = blob.getKey(); - S3ObjectSummary objectSummary = new S3ObjectSummary(); - - if (prefix.isEmpty() || blobName.startsWith(prefix)) { - objectSummary.setKey(blobName); + public static AmazonS3 createClient(final String bucket, + final boolean serverSideEncryption, + final String cannedACL, + final String storageClass) { + + final AmazonS3 mockedClient = mock(AmazonS3.class); + + // Create or erase the bucket + BUCKETS.put(bucket, new ConcurrentHashMap<>()); + + // Bucket exists? + when(mockedClient.doesBucketExist(any(String.class))).thenAnswer(invocation -> { + String bucketName = (String) invocation.getArguments()[0]; + assertThat(bucketName, equalTo(bucket)); + return BUCKETS.containsKey(bucketName); + }); + + // Blob exists? + when(mockedClient.doesObjectExist(any(String.class), any(String.class))).thenAnswer(invocation -> { + String bucketName = (String) invocation.getArguments()[0]; + String objectName = (String) invocation.getArguments()[1]; + assertThat(bucketName, equalTo(bucket)); + return BUCKETS.getOrDefault(bucketName, emptyMap()).containsKey(objectName); + }); + + // Write blob + when(mockedClient.putObject(any(PutObjectRequest.class))).thenAnswer(invocation -> { + PutObjectRequest request = (PutObjectRequest) invocation.getArguments()[0]; + assertThat(request.getBucketName(), equalTo(bucket)); + assertThat(request.getMetadata().getSSEAlgorithm(), serverSideEncryption ? equalTo("AES256") : nullValue()); + assertThat(request.getCannedAcl(), notNullValue()); + assertThat(request.getCannedAcl().toString(), cannedACL != null ? equalTo(cannedACL) : equalTo("private")); + assertThat(request.getStorageClass(), storageClass != null ? equalTo(storageClass) : equalTo("STANDARD")); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(request.getInputStream(), out); + assertThat((long) out.size(), equalTo(request.getMetadata().getContentLength())); + + BUCKETS.computeIfAbsent(request.getBucketName(), s -> new ConcurrentHashMap<>()).put(request.getKey(), out.toByteArray()); + return null; + }); + + // Read blob + when(mockedClient.getObject(any(String.class), any(String.class))).thenAnswer(invocation -> { + String bucketName = (String) invocation.getArguments()[0]; + String objectName = (String) invocation.getArguments()[1]; + assertThat(bucketName, equalTo(bucket)); + + byte[] blob = BUCKETS.getOrDefault(bucketName, emptyMap()).get(objectName); + if(blob == null){ + AmazonS3Exception exception = new AmazonS3Exception("Blob not found"); + exception.setStatusCode(404); + throw exception; + } - try { - objectSummary.setSize(getSize(blob.getValue())); - } catch (IOException e) { - throw new AmazonS3Exception("Object listing " + - "failed for blob [" + blob.getKey() + "]"); + S3Object response = new S3Object(); + response.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(blob), null, false)); + return response; + }); + + // Copy blob + when(mockedClient.copyObject(any(CopyObjectRequest.class))).thenAnswer(invocation -> { + CopyObjectRequest request = (CopyObjectRequest) invocation.getArguments()[0]; + assertThat(request.getSourceBucketName(), equalTo(bucket)); + assertThat(request.getDestinationBucketName(), equalTo(bucket)); + + Map blobsInBucket = BUCKETS.getOrDefault(bucket, emptyMap()); + byte[] blob = blobsInBucket.get(request.getSourceKey()); + if(blob != null) { + blobsInBucket.put(request.getDestinationKey(), blob); + } else { + AmazonS3Exception exception = new AmazonS3Exception("Blob not found"); + exception.setStatusCode(404); + throw exception; + } + return null; + }); + + // List BUCKETS + when(mockedClient.listObjects(any(String.class), any(String.class))).thenAnswer(invocation -> { + String bucketName = (String) invocation.getArguments()[0]; + String prefix = (String) invocation.getArguments()[1]; + + assertThat(bucketName, equalTo(bucket)); + ObjectListing listing = new ObjectListing(); + listing.setBucketName(bucketName); + listing.setPrefix(prefix); + for (Map.Entry blob : BUCKETS.getOrDefault(bucketName, emptyMap()).entrySet()) { + if (blob.getKey().startsWith(prefix)) { + S3ObjectSummary summary = new S3ObjectSummary(); + summary.setBucketName(bucketName); + summary.setKey(blob.getKey()); + summary.setSize(blob.getValue().length); + listing.getObjectSummaries().add(summary); } - - mockObjectSummaries.add(objectSummary); } - } - - list.setObjectSummaries(mockObjectSummaries); - return list; - } - - @Override - public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) - throws AmazonClientException, AmazonServiceException { - simulateS3SocketConnection(); - String sourceBlobName = copyObjectRequest.getSourceKey(); - String targetBlobName = copyObjectRequest.getDestinationKey(); - - if (!blobs.containsKey(sourceBlobName)) { - throw new AmazonS3Exception("Source blob [" + - sourceBlobName + "] does not exist."); - } - - if (blobs.containsKey(targetBlobName)) { - throw new AmazonS3Exception("Target blob [" + - targetBlobName + "] already exists."); - } - - blobs.put(targetBlobName, blobs.get(sourceBlobName)); - return new CopyObjectResult(); // nothing is done with it - } - - @Override - public void deleteObject(DeleteObjectRequest deleteObjectRequest) - throws AmazonClientException, AmazonServiceException { - simulateS3SocketConnection(); - String blobName = deleteObjectRequest.getKey(); - - if (!blobs.containsKey(blobName)) { - throw new AmazonS3Exception("[" + blobName + "] does not exist."); - } - - blobs.remove(blobName); - } - - private int getSize(InputStream stream) throws IOException { - int size = stream.read(byteCounter); - stream.reset(); // in case we ever need the size again - return size; - } - - private class MockObjectListing extends ObjectListing { - // the objectSummaries attribute in ObjectListing.java - // is read-only, but we need to be able to write to it, - // so we create a mock of it to work around this - private List mockObjectSummaries; - - @Override - public List getObjectSummaries() { - return mockObjectSummaries; - } + return listing; + }); + + // List next batch of BUCKETS + when(mockedClient.listNextBatchOfObjects(any(ObjectListing.class))).thenAnswer(invocation -> { + ObjectListing objectListing = (ObjectListing) invocation.getArguments()[0]; + assertThat(objectListing.getBucketName(), equalTo(bucket)); + return new ObjectListing(); + }); + + // Delete blob + doAnswer(invocation -> { + String bucketName = (String) invocation.getArguments()[0]; + String objectName = (String) invocation.getArguments()[1]; + assertThat(bucketName, equalTo(bucket)); + + Map blobsInBucket = BUCKETS.getOrDefault(bucketName, emptyMap()); + if(blobsInBucket.remove(objectName) == null){ + AmazonS3Exception exception = new AmazonS3Exception("Blob not found"); + exception.setStatusCode(404); + throw exception; + } + return null; + }).when(mockedClient).deleteObject(any(String.class), any(String.class)); + + // Delete multiple BUCKETS + doAnswer(invocation -> { + DeleteObjectsRequest deleteObjectsRequest = (DeleteObjectsRequest) invocation.getArguments()[0]; + assertThat(deleteObjectsRequest.getBucketName(), equalTo(bucket)); + + Map blobsInBucket = BUCKETS.getOrDefault(deleteObjectsRequest.getBucketName(), emptyMap()); + for (DeleteObjectsRequest.KeyVersion key : deleteObjectsRequest.getKeys()) { + if(blobsInBucket.remove(key.getKey()) == null){ + AmazonS3Exception exception = new AmazonS3Exception("Blob not found"); + exception.setStatusCode(404); + throw exception; + } + } + return null; + }).when(mockedClient).deleteObjects(any(DeleteObjectsRequest.class)); - private void setObjectSummaries(List objectSummaries) { - mockObjectSummaries = objectSummaries; - } + return mockedClient; } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java index 5998540e7a8fa..453ef3213f0b6 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java @@ -37,26 +37,19 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.mocksocket.MockServerSocket; import org.elasticsearch.repositories.ESBlobStoreContainerTestCase; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.mockito.ArgumentCaptor; import java.io.ByteArrayInputStream; import java.io.IOException; -import java.net.InetAddress; -import java.net.ServerSocket; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.repositories.s3.S3BlobStoreTests.randomMockS3BlobStore; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doNothing; @@ -67,36 +60,11 @@ public class S3BlobStoreContainerTests extends ESBlobStoreContainerTestCase { - private static ServerSocket mockS3ServerSocket; - - private static Thread mockS3AcceptorThread; - - // Opens a MockSocket to simulate connections to S3 checking that SocketPermissions are set up correctly. - // See MockAmazonS3.simulateS3SocketConnection. - @BeforeClass - public static void openMockSocket() throws IOException { - mockS3ServerSocket = new MockServerSocket(0, 50, InetAddress.getByName("127.0.0.1")); - mockS3AcceptorThread = new Thread(() -> { - while (!mockS3ServerSocket.isClosed()) { - try { - // Accept connections from MockAmazonS3. - mockS3ServerSocket.accept(); - } catch (IOException e) { - } - } - }); - mockS3AcceptorThread.start(); + protected BlobStore newBlobStore() { + return randomMockS3BlobStore(); } - protected BlobStore newBlobStore() throws IOException { - MockAmazonS3 client = new MockAmazonS3(mockS3ServerSocket.getLocalPort()); - String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); - - return new S3BlobStore(Settings.EMPTY, client, bucket, false, - new ByteSizeValue(10, ByteSizeUnit.MB), "public-read-write", "standard"); - } - - public void testExecuteSingleUploadBlobSizeTooLarge() throws IOException { + public void testExecuteSingleUploadBlobSizeTooLarge() { final long blobSize = ByteSizeUnit.GB.toBytes(randomIntBetween(6, 10)); final S3BlobStore blobStore = mock(S3BlobStore.class); final S3BlobContainer blobContainer = new S3BlobContainer(mock(BlobPath.class), blobStore); @@ -106,7 +74,7 @@ public void testExecuteSingleUploadBlobSizeTooLarge() throws IOException { assertEquals("Upload request size [" + blobSize + "] can't be larger than 5gb", e.getMessage()); } - public void testExecuteSingleUploadBlobSizeLargerThanBufferSize() throws IOException { + public void testExecuteSingleUploadBlobSizeLargerThanBufferSize() { final S3BlobStore blobStore = mock(S3BlobStore.class); when(blobStore.bufferSizeInBytes()).thenReturn(ByteSizeUnit.MB.toBytes(1)); @@ -168,7 +136,7 @@ public void testExecuteSingleUpload() throws IOException { } } - public void testExecuteMultipartUploadBlobSizeTooLarge() throws IOException { + public void testExecuteMultipartUploadBlobSizeTooLarge() { final long blobSize = ByteSizeUnit.TB.toBytes(randomIntBetween(6, 10)); final S3BlobStore blobStore = mock(S3BlobStore.class); final S3BlobContainer blobContainer = new S3BlobContainer(mock(BlobPath.class), blobStore); @@ -179,7 +147,7 @@ public void testExecuteMultipartUploadBlobSizeTooLarge() throws IOException { assertEquals("Multipart upload request size [" + blobSize + "] can't be larger than 5tb", e.getMessage()); } - public void testExecuteMultipartUploadBlobSizeTooSmall() throws IOException { + public void testExecuteMultipartUploadBlobSizeTooSmall() { final long blobSize = ByteSizeUnit.MB.toBytes(randomIntBetween(1, 4)); final S3BlobStore blobStore = mock(S3BlobStore.class); final S3BlobContainer blobContainer = new S3BlobContainer(mock(BlobPath.class), blobStore); @@ -291,7 +259,7 @@ public void testExecuteMultipartUpload() throws IOException { assertEquals(expectedEtags, actualETags); } - public void testExecuteMultipartUploadAborted() throws IOException { + public void testExecuteMultipartUploadAborted() { final String bucketName = randomAlphaOfLengthBetween(1, 10); final String blobName = randomAlphaOfLengthBetween(1, 10); final BlobPath blobPath = new BlobPath(); @@ -418,12 +386,4 @@ private static void assertNumberOfMultiparts(final int expectedParts, final long assertEquals("Expected number of parts [" + expectedParts + "] but got [" + result.v1() + "]", expectedParts, (long) result.v1()); assertEquals("Expected remaining [" + expectedRemaining + "] but got [" + result.v2() + "]", expectedRemaining, (long) result.v2()); } - - @AfterClass - public static void closeMockSocket() throws IOException, InterruptedException { - mockS3ServerSocket.close(); - mockS3AcceptorThread.join(); - mockS3AcceptorThread = null; - mockS3ServerSocket = null; - } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java new file mode 100644 index 0000000000000..0cd29a2303404 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.s3; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.StorageClass; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase; +import org.junit.BeforeClass; + +import java.util.Collection; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCase { + + private static String bucket; + private static String client; + private static ByteSizeValue bufferSize; + private static boolean serverSideEncryption; + private static String cannedACL; + private static String storageClass; + + @BeforeClass + public static void setUpRepositorySettings() { + bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); + client = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); + bufferSize = new ByteSizeValue(randomIntBetween(5, 50), ByteSizeUnit.MB); + serverSideEncryption = randomBoolean(); + if (randomBoolean()) { + cannedACL = randomFrom(CannedAccessControlList.values()).toString(); + } + if (randomBoolean()) { + storageClass = randomValueOtherThan(StorageClass.Glacier, () -> randomFrom(StorageClass.values())).toString(); + } + } + + @Override + protected void createTestRepository(final String name) { + assertAcked(client().admin().cluster().preparePutRepository(name) + .setType(S3Repository.TYPE) + .setSettings(Settings.builder() + .put(S3Repository.BUCKET_SETTING.getKey(), bucket) + .put(InternalAwsS3Service.CLIENT_NAME.getKey(), client) + .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), bufferSize) + .put(S3Repository.SERVER_SIDE_ENCRYPTION_SETTING.getKey(), serverSideEncryption) + .put(S3Repository.CANNED_ACL_SETTING.getKey(), cannedACL) + .put(S3Repository.STORAGE_CLASS_SETTING.getKey(), storageClass))); + } + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(TestS3RepositoryPlugin.class); + } + + public static class TestS3RepositoryPlugin extends S3RepositoryPlugin { + + public TestS3RepositoryPlugin(final Settings settings) { + super(settings); + } + + @Override + public Map getRepositories(final Environment env, final NamedXContentRegistry registry) { + return Collections.singletonMap(S3Repository.TYPE, (metadata) -> + new S3Repository(metadata, env.settings(), registry, new InternalAwsS3Service(env.settings(), emptyMap()) { + @Override + public synchronized AmazonS3 client(final Settings repositorySettings) { + return MockAmazonS3.createClient(bucket, serverSideEncryption, cannedACL, storageClass); + } + })); + } + } +} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java index 17bea5239fe7e..bb32c106e734a 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java @@ -19,18 +19,28 @@ package org.elasticsearch.repositories.s3; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.StorageClass; +import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreException; -import org.elasticsearch.repositories.s3.S3BlobStore; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.repositories.ESBlobStoreTestCase; -import java.io.IOException; +import java.util.Locale; import static org.hamcrest.Matchers.equalTo; -public class S3BlobStoreTests extends ESTestCase { - public void testInitCannedACL() throws IOException { +public class S3BlobStoreTests extends ESBlobStoreTestCase { + + @Override + protected BlobStore newBlobStore() { + return randomMockS3BlobStore(); + } + + public void testInitCannedACL() { String[] aclList = new String[]{ "private", "public-read", "public-read-write", "authenticated-read", "log-delivery-write", "bucket-owner-read", "bucket-owner-full-control"}; @@ -52,16 +62,12 @@ public void testInitCannedACL() throws IOException { } } - public void testInvalidCannedACL() throws IOException { - try { - S3BlobStore.initCannedACL("test_invalid"); - fail("CannedACL should fail"); - } catch (BlobStoreException ex) { - assertThat(ex.getMessage(), equalTo("cannedACL is not valid: [test_invalid]")); - } + public void testInvalidCannedACL() { + BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initCannedACL("test_invalid")); + assertThat(ex.getMessage(), equalTo("cannedACL is not valid: [test_invalid]")); } - public void testInitStorageClass() throws IOException { + public void testInitStorageClass() { // it should default to `standard` assertThat(S3BlobStore.initStorageClass(null), equalTo(StorageClass.Standard)); assertThat(S3BlobStore.initStorageClass(""), equalTo(StorageClass.Standard)); @@ -72,25 +78,44 @@ public void testInitStorageClass() throws IOException { assertThat(S3BlobStore.initStorageClass("reduced_redundancy"), equalTo(StorageClass.ReducedRedundancy)); } - public void testCaseInsensitiveStorageClass() throws IOException { + public void testCaseInsensitiveStorageClass() { assertThat(S3BlobStore.initStorageClass("sTandaRd"), equalTo(StorageClass.Standard)); assertThat(S3BlobStore.initStorageClass("sTandaRd_Ia"), equalTo(StorageClass.StandardInfrequentAccess)); assertThat(S3BlobStore.initStorageClass("reduCED_redundancy"), equalTo(StorageClass.ReducedRedundancy)); } - public void testInvalidStorageClass() throws IOException { - try { - S3BlobStore.initStorageClass("whatever"); - } catch(BlobStoreException ex) { - assertThat(ex.getMessage(), equalTo("`whatever` is not a valid S3 Storage Class.")); - } + public void testInvalidStorageClass() { + BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initStorageClass("whatever")); + assertThat(ex.getMessage(), equalTo("`whatever` is not a valid S3 Storage Class.")); + } + + public void testRejectGlacierStorageClass() { + BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initStorageClass("glacier")); + assertThat(ex.getMessage(), equalTo("Glacier storage class is not supported")); } - public void testRejectGlacierStorageClass() throws IOException { - try { - S3BlobStore.initStorageClass("glacier"); - } catch(BlobStoreException ex) { - assertThat(ex.getMessage(), equalTo("Glacier storage class is not supported")); + /** + * Creates a new {@link S3BlobStore} with random settings. + *

+ * The blobstore uses internally a mocked {@link AmazonS3} client. + */ + public static S3BlobStore randomMockS3BlobStore() { + String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); + ByteSizeValue bufferSize = new ByteSizeValue(randomIntBetween(5, 100), ByteSizeUnit.MB); + boolean serverSideEncryption = randomBoolean(); + + String cannedACL = null; + if (randomBoolean()) { + cannedACL = randomFrom(CannedAccessControlList.values()).toString(); + } + + String storageClass = null; + if (randomBoolean()) { + storageClass = randomValueOtherThan(StorageClass.Glacier, () -> randomFrom(StorageClass.values())).toString(); } + + AmazonS3 client = MockAmazonS3.createClient(bucket, serverSideEncryption, cannedACL, storageClass); + + return new S3BlobStore(Settings.EMPTY, client, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass); } } From 41bb953123bcb7d0d550a5f863c8ee354d295c31 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 4 Apr 2018 17:49:33 +0200 Subject: [PATCH 2/3] Apply feedback --- .../repositories/s3/MockAmazonS3.java | 281 +++++++++--------- .../s3/S3BlobStoreRepositoryTests.java | 5 +- .../repositories/s3/S3BlobStoreTests.java | 6 +- 3 files changed, 150 insertions(+), 142 deletions(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java index fe845398f8dce..bc6a041baa223 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java @@ -19,177 +19,182 @@ package org.elasticsearch.repositories.s3; +import com.amazonaws.AmazonClientException; +import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AbstractAmazonS3; -import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.CopyObjectResult; +import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Objects; +import java.util.concurrent.ConcurrentMap; -import static java.util.Collections.emptyMap; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; class MockAmazonS3 extends AbstractAmazonS3 { - static final Map> BUCKETS = new ConcurrentHashMap<>(); + private final ConcurrentMap blobs; + private final String bucket; + private final boolean serverSideEncryption; + private final String cannedACL; + private final String storageClass; + + MockAmazonS3(final ConcurrentMap blobs, + final String bucket, + final boolean serverSideEncryption, + final String cannedACL, + final String storageClass) { + this.blobs = Objects.requireNonNull(blobs); + this.bucket = Objects.requireNonNull(bucket); + this.serverSideEncryption = serverSideEncryption; + this.cannedACL = cannedACL; + this.storageClass = storageClass; + } + + @Override + public boolean doesBucketExist(final String bucket) { + return this.bucket.equalsIgnoreCase(bucket); + } - private MockAmazonS3() { + @Override + public boolean doesObjectExist(final String bucketName, final String objectName) throws SdkClientException { + assertThat(bucketName, equalTo(bucket)); + return blobs.containsKey(objectName); } - public static AmazonS3 createClient(final String bucket, - final boolean serverSideEncryption, - final String cannedACL, - final String storageClass) { - - final AmazonS3 mockedClient = mock(AmazonS3.class); - - // Create or erase the bucket - BUCKETS.put(bucket, new ConcurrentHashMap<>()); - - // Bucket exists? - when(mockedClient.doesBucketExist(any(String.class))).thenAnswer(invocation -> { - String bucketName = (String) invocation.getArguments()[0]; - assertThat(bucketName, equalTo(bucket)); - return BUCKETS.containsKey(bucketName); - }); - - // Blob exists? - when(mockedClient.doesObjectExist(any(String.class), any(String.class))).thenAnswer(invocation -> { - String bucketName = (String) invocation.getArguments()[0]; - String objectName = (String) invocation.getArguments()[1]; - assertThat(bucketName, equalTo(bucket)); - return BUCKETS.getOrDefault(bucketName, emptyMap()).containsKey(objectName); - }); - - // Write blob - when(mockedClient.putObject(any(PutObjectRequest.class))).thenAnswer(invocation -> { - PutObjectRequest request = (PutObjectRequest) invocation.getArguments()[0]; - assertThat(request.getBucketName(), equalTo(bucket)); - assertThat(request.getMetadata().getSSEAlgorithm(), serverSideEncryption ? equalTo("AES256") : nullValue()); - assertThat(request.getCannedAcl(), notNullValue()); - assertThat(request.getCannedAcl().toString(), cannedACL != null ? equalTo(cannedACL) : equalTo("private")); - assertThat(request.getStorageClass(), storageClass != null ? equalTo(storageClass) : equalTo("STANDARD")); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); + @Override + public PutObjectResult putObject(final PutObjectRequest request) throws AmazonClientException { + assertThat(request.getBucketName(), equalTo(bucket)); + assertThat(request.getBucketName(), equalTo(bucket)); + assertThat(request.getMetadata().getSSEAlgorithm(), serverSideEncryption ? equalTo("AES256") : nullValue()); + assertThat(request.getCannedAcl(), notNullValue()); + assertThat(request.getCannedAcl().toString(), cannedACL != null ? equalTo(cannedACL) : equalTo("private")); + assertThat(request.getStorageClass(), storageClass != null ? equalTo(storageClass) : equalTo("STANDARD")); + + + final String blobName = request.getKey(); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { Streams.copy(request.getInputStream(), out); - assertThat((long) out.size(), equalTo(request.getMetadata().getContentLength())); + blobs.put(blobName, out.toByteArray()); + } catch (IOException e) { + throw new AmazonClientException(e); + } + return new PutObjectResult(); + } - BUCKETS.computeIfAbsent(request.getBucketName(), s -> new ConcurrentHashMap<>()).put(request.getKey(), out.toByteArray()); - return null; - }); + @Override + public S3Object getObject(final GetObjectRequest request) throws AmazonClientException { + assertThat(request.getBucketName(), equalTo(bucket)); - // Read blob - when(mockedClient.getObject(any(String.class), any(String.class))).thenAnswer(invocation -> { - String bucketName = (String) invocation.getArguments()[0]; - String objectName = (String) invocation.getArguments()[1]; - assertThat(bucketName, equalTo(bucket)); + final String blobName = request.getKey(); + final byte[] content = blobs.get(blobName); + if (content == null) { + AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist."); + exception.setStatusCode(404); + throw exception; + } - byte[] blob = BUCKETS.getOrDefault(bucketName, emptyMap()).get(objectName); - if(blob == null){ - AmazonS3Exception exception = new AmazonS3Exception("Blob not found"); - exception.setStatusCode(404); - throw exception; - } + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(content.length); - S3Object response = new S3Object(); - response.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(blob), null, false)); - return response; - }); - - // Copy blob - when(mockedClient.copyObject(any(CopyObjectRequest.class))).thenAnswer(invocation -> { - CopyObjectRequest request = (CopyObjectRequest) invocation.getArguments()[0]; - assertThat(request.getSourceBucketName(), equalTo(bucket)); - assertThat(request.getDestinationBucketName(), equalTo(bucket)); - - Map blobsInBucket = BUCKETS.getOrDefault(bucket, emptyMap()); - byte[] blob = blobsInBucket.get(request.getSourceKey()); - if(blob != null) { - blobsInBucket.put(request.getDestinationKey(), blob); - } else { - AmazonS3Exception exception = new AmazonS3Exception("Blob not found"); - exception.setStatusCode(404); - throw exception; - } - return null; - }); - - // List BUCKETS - when(mockedClient.listObjects(any(String.class), any(String.class))).thenAnswer(invocation -> { - String bucketName = (String) invocation.getArguments()[0]; - String prefix = (String) invocation.getArguments()[1]; - - assertThat(bucketName, equalTo(bucket)); - ObjectListing listing = new ObjectListing(); - listing.setBucketName(bucketName); - listing.setPrefix(prefix); - for (Map.Entry blob : BUCKETS.getOrDefault(bucketName, emptyMap()).entrySet()) { - if (blob.getKey().startsWith(prefix)) { - S3ObjectSummary summary = new S3ObjectSummary(); - summary.setBucketName(bucketName); - summary.setKey(blob.getKey()); - summary.setSize(blob.getValue().length); - listing.getObjectSummaries().add(summary); - } + S3Object s3Object = new S3Object(); + s3Object.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(content), null, false)); + s3Object.setKey(blobName); + s3Object.setObjectMetadata(metadata); + + return s3Object; + } + + @Override + public ObjectListing listObjects(final ListObjectsRequest request) throws AmazonClientException { + assertThat(request.getBucketName(), equalTo(bucket)); + + final ObjectListing listing = new ObjectListing(); + listing.setBucketName(request.getBucketName()); + listing.setPrefix(request.getPrefix()); + + for (Map.Entry blob : blobs.entrySet()) { + if (Strings.isEmpty(request.getPrefix()) || blob.getKey().startsWith(request.getPrefix())) { + S3ObjectSummary summary = new S3ObjectSummary(); + summary.setBucketName(request.getBucketName()); + summary.setKey(blob.getKey()); + summary.setSize(blob.getValue().length); + listing.getObjectSummaries().add(summary); } - return listing; - }); - - // List next batch of BUCKETS - when(mockedClient.listNextBatchOfObjects(any(ObjectListing.class))).thenAnswer(invocation -> { - ObjectListing objectListing = (ObjectListing) invocation.getArguments()[0]; - assertThat(objectListing.getBucketName(), equalTo(bucket)); - return new ObjectListing(); - }); - - // Delete blob - doAnswer(invocation -> { - String bucketName = (String) invocation.getArguments()[0]; - String objectName = (String) invocation.getArguments()[1]; - assertThat(bucketName, equalTo(bucket)); - - Map blobsInBucket = BUCKETS.getOrDefault(bucketName, emptyMap()); - if(blobsInBucket.remove(objectName) == null){ - AmazonS3Exception exception = new AmazonS3Exception("Blob not found"); + } + return listing; + } + + @Override + public CopyObjectResult copyObject(final CopyObjectRequest request) throws AmazonClientException { + assertThat(request.getSourceBucketName(), equalTo(bucket)); + assertThat(request.getDestinationBucketName(), equalTo(bucket)); + + final String sourceBlobName = request.getSourceKey(); + + final byte[] content = blobs.get(sourceBlobName); + if (content == null) { + AmazonS3Exception exception = new AmazonS3Exception("[" + sourceBlobName + "] does not exist."); + exception.setStatusCode(404); + throw exception; + } + + blobs.put(request.getDestinationKey(), content); + return new CopyObjectResult(); + } + + @Override + public void deleteObject(final DeleteObjectRequest request) throws AmazonClientException { + assertThat(request.getBucketName(), equalTo(bucket)); + + final String blobName = request.getKey(); + if (blobs.remove(blobName) == null) { + AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist."); + exception.setStatusCode(404); + throw exception; + } + } + + @Override + public DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) throws SdkClientException { + assertThat(request.getBucketName(), equalTo(bucket)); + + final List deletions = new ArrayList<>(); + for (DeleteObjectsRequest.KeyVersion key : request.getKeys()) { + if(blobs.remove(key.getKey()) == null){ + AmazonS3Exception exception = new AmazonS3Exception("[" + key + "] does not exist."); exception.setStatusCode(404); throw exception; + } else { + DeleteObjectsResult.DeletedObject deletion = new DeleteObjectsResult.DeletedObject(); + deletion.setKey(key.getKey()); + deletions.add(deletion); } - return null; - }).when(mockedClient).deleteObject(any(String.class), any(String.class)); - - // Delete multiple BUCKETS - doAnswer(invocation -> { - DeleteObjectsRequest deleteObjectsRequest = (DeleteObjectsRequest) invocation.getArguments()[0]; - assertThat(deleteObjectsRequest.getBucketName(), equalTo(bucket)); - - Map blobsInBucket = BUCKETS.getOrDefault(deleteObjectsRequest.getBucketName(), emptyMap()); - for (DeleteObjectsRequest.KeyVersion key : deleteObjectsRequest.getKeys()) { - if(blobsInBucket.remove(key.getKey()) == null){ - AmazonS3Exception exception = new AmazonS3Exception("Blob not found"); - exception.setStatusCode(404); - throw exception; - } - } - return null; - }).when(mockedClient).deleteObjects(any(DeleteObjectsRequest.class)); - - return mockedClient; + } + return new DeleteObjectsResult(deletions); } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 0cd29a2303404..db07800ebb151 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -35,12 +35,15 @@ import java.util.Collections; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import static java.util.Collections.emptyMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; public class S3BlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCase { + private static final ConcurrentMap blobs = new ConcurrentHashMap<>(); private static String bucket; private static String client; private static ByteSizeValue bufferSize; @@ -92,7 +95,7 @@ public Map getRepositories(final Environment env, fi new S3Repository(metadata, env.settings(), registry, new InternalAwsS3Service(env.settings(), emptyMap()) { @Override public synchronized AmazonS3 client(final Settings repositorySettings) { - return MockAmazonS3.createClient(bucket, serverSideEncryption, cannedACL, storageClass); + return new MockAmazonS3(blobs, bucket, serverSideEncryption, cannedACL, storageClass); } })); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java index bb32c106e734a..4a23e4efa9a29 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.repositories.ESBlobStoreTestCase; import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; import static org.hamcrest.Matchers.equalTo; @@ -97,7 +98,7 @@ public void testRejectGlacierStorageClass() { /** * Creates a new {@link S3BlobStore} with random settings. *

- * The blobstore uses internally a mocked {@link AmazonS3} client. + * The blobstore uses a {@link MockAmazonS3} client. */ public static S3BlobStore randomMockS3BlobStore() { String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); @@ -114,8 +115,7 @@ public static S3BlobStore randomMockS3BlobStore() { storageClass = randomValueOtherThan(StorageClass.Glacier, () -> randomFrom(StorageClass.values())).toString(); } - AmazonS3 client = MockAmazonS3.createClient(bucket, serverSideEncryption, cannedACL, storageClass); - + AmazonS3 client = new MockAmazonS3(new ConcurrentHashMap<>(), bucket, serverSideEncryption, cannedACL, storageClass); return new S3BlobStore(Settings.EMPTY, client, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass); } } From 6cd10405ebd7097a67111fba6cd3f7125effadd8 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 5 Apr 2018 10:11:56 +0200 Subject: [PATCH 3/3] Apply feedback --- .../org/elasticsearch/repositories/s3/MockAmazonS3.java | 3 +-- .../repositories/s3/S3BlobStoreRepositoryTests.java | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java index bc6a041baa223..caa1c0b467e52 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java @@ -87,7 +87,6 @@ public boolean doesObjectExist(final String bucketName, final String objectName) @Override public PutObjectResult putObject(final PutObjectRequest request) throws AmazonClientException { - assertThat(request.getBucketName(), equalTo(bucket)); assertThat(request.getBucketName(), equalTo(bucket)); assertThat(request.getMetadata().getSSEAlgorithm(), serverSideEncryption ? equalTo("AES256") : nullValue()); assertThat(request.getCannedAcl(), notNullValue()); @@ -185,7 +184,7 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) throws Sd final List deletions = new ArrayList<>(); for (DeleteObjectsRequest.KeyVersion key : request.getKeys()) { - if(blobs.remove(key.getKey()) == null){ + if (blobs.remove(key.getKey()) == null) { AmazonS3Exception exception = new AmazonS3Exception("[" + key + "] does not exist."); exception.setStatusCode(404); throw exception; diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index db07800ebb151..e3e89c41514de 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase; +import org.junit.AfterClass; import org.junit.BeforeClass; import java.util.Collection; @@ -65,6 +66,11 @@ public static void setUpRepositorySettings() { } } + @AfterClass + public static void wipeRepository() { + blobs.clear(); + } + @Override protected void createTestRepository(final String name) { assertAcked(client().admin().cluster().preparePutRepository(name)