Raised IOException on deleteBlob

core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java

@@ -85,7 +85,7 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws
     @Override
     public void deleteBlob(String blobName) throws IOException {
         Path blobPath = path.resolve(blobName);
-        Files.deleteIfExists(blobPath);
+        Files.delete(blobPath);
     }
 
     @Override

plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java

@@ -70,6 +70,11 @@ public boolean blobExists(String blobName) {
     @Override
     public InputStream readBlob(String blobName) throws IOException {
         logger.trace("readBlob({})", blobName);
+
+        if (!blobExists(blobName)) {
+            throw new IOException("Blob [" + blobName + "] does not exist");
+        }
+
         try {
             return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
         } catch (StorageException e) {
@@ -116,6 +121,11 @@ private OutputStream createOutput(String blobName) throws IOException {
     @Override
     public void deleteBlob(String blobName) throws IOException {
         logger.trace("deleteBlob({})", blobName);
+
+        if (!blobExists(blobName)) {
+            throw new IOException("Blob [" + blobName + "] does not exist");
+        }
+
         try {
             blobStore.deleteBlob(blobStore.container(), buildKey(blobName));
         } catch (URISyntaxException | StorageException e) {

plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java

@@ -95,13 +95,13 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode
         MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
         for (String blobName : blobs.keySet()) {
             final String checkBlob;
-            if (keyPath != null) {
+            if (keyPath != null && !keyPath.isEmpty()) {
                 // strip off key path from the beginning of the blob name
                 checkBlob = blobName.replace(keyPath, "");
             } else {
                 checkBlob = blobName;
             }
-            if (startsWithIgnoreCase(checkBlob, prefix)) {
+            if (prefix == null || startsWithIgnoreCase(checkBlob, prefix)) {
                 blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, blobs.get(blobName).size()));
             }
         }

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import com.microsoft.azure.storage.StorageException;
+import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
+import org.elasticsearch.cloud.azure.storage.AzureStorageServiceMock;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
+import org.elasticsearch.repositories.RepositoryName;
+import org.elasticsearch.repositories.RepositorySettings;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+public class AzureBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
+    @Override
+    protected BlobStore newBlobStore() throws IOException {
+        try {
+            RepositoryName repositoryName = new RepositoryName("azure", "ittest");
+            RepositorySettings repositorySettings = new RepositorySettings(
+                    Settings.EMPTY, Settings.EMPTY);
+            AzureStorageServiceMock client = new AzureStorageServiceMock(Settings.EMPTY);
+            return new AzureBlobStore(repositoryName, Settings.EMPTY, repositorySettings, client);
+        } catch (URISyntaxException | StorageException e) {
+            throw new IOException(e);
+        }
+    }
+}

plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java

@@ -68,16 +68,16 @@ public Boolean run(FileContext fileContext) throws IOException {
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
-        try {
-            store.execute(new Operation<Boolean>() {
-                @Override
-                public Boolean run(FileContext fileContext) throws IOException {
-                    return fileContext.delete(new Path(path, blobName), true);
-                }
-            });
-        } catch (FileNotFoundException ok) {
-            // behaves like Files.deleteIfExists
+        if (!blobExists(blobName)) {
+            throw new IOException("Blob [" + blobName + "] does not exist");
         }
+
+        store.execute(new Operation<Boolean>() {
+            @Override
+            public Boolean run(FileContext fileContext) throws IOException {
+                return fileContext.delete(new Path(path, blobName), true);
+            }
+        });
     }
 
     @Override

plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
+
+import javax.security.auth.Subject;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.AccessController;
+import java.security.Principal;
+import java.security.PrivilegedAction;
+import java.util.Collections;
+
+public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
+
+    @Override
+    protected BlobStore newBlobStore() throws IOException {
+        return AccessController.doPrivileged(
+                new PrivilegedAction<HdfsBlobStore>() {
+                    @Override
+                    public HdfsBlobStore run() {
+                        try {
+                            FileContext fileContext = createContext(new URI("hdfs:///"));
+                            return new HdfsBlobStore(fileContext, "temp", 1024);
+                        } catch (IOException | URISyntaxException e) {
+                            throw new RuntimeException(e);
+                        }
+            }
+        });
+    }
+
+    public FileContext createContext(URI uri) {
+        // mirrors HdfsRepository.java behaviour
+        Configuration cfg = new Configuration(true);
+        cfg.setClassLoader(HdfsRepository.class.getClassLoader());
+        cfg.reloadConfiguration();
+
+        Constructor<?> ctor;
+        Subject subject;
+
+        try {
+            Class<?> clazz = Class.forName("org.apache.hadoop.security.User");
+            ctor = clazz.getConstructor(String.class);
+            ctor.setAccessible(true);
+        }  catch (ClassNotFoundException | NoSuchMethodException e) {
+            throw new RuntimeException(e);
+        }
+
+        try {
+            Principal principal = (Principal) ctor.newInstance(System.getProperty("user.name"));
+            subject = new Subject(false, Collections.singleton(principal),
+                    Collections.emptySet(), Collections.emptySet());
+        } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+
+        // disable file system cache
+        cfg.setBoolean("fs.hdfs.impl.disable.cache", true);
+
+        // set file system to TestingFs to avoid a bunch of security
+        // checks, similar to what is done in HdfsTests.java
+        cfg.set(String.format("fs.AbstractFileSystem.%s.impl", uri.getScheme()),
+                TestingFs.class.getName());
+
+        // create the FileContext with our user
+        return Subject.doAs(subject, new PrivilegedAction<FileContext>() {
+            @Override
+            public FileContext run() {
+                try {
+                    TestingFs fs = (TestingFs) AbstractFileSystem.get(uri, cfg);
+                    return FileContext.getFileContext(fs, cfg);
+                } catch (UnsupportedFileSystemException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
+    }
+}

plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java

@@ -108,6 +108,10 @@ public void writeBlob(String blobName, BytesReference bytes) throws IOException
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
+        if (!blobExists(blobName)) {
+            throw new IOException("Blob [" + blobName + "] does not exist");
+        }
+
         try {
             blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName));
         } catch (AmazonClientException e) {