From ad4f704a4f9e891f020df146f77abd706c5e28e6 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 17 Jan 2017 18:46:49 -0600 Subject: [PATCH 01/24] Add basic url repository module --- .../common/settings/ClusterSettings.java | 6 +-- .../repositories/RepositoriesModule.java | 13 ++--- modules/url-repository/build.gradle | 33 +++++++++++++ .../blobstore/url/URLBlobContainer.java | 0 .../common/blobstore/url/URLBlobStore.java | 0 .../repositories/URLRespositoryPlugin.java | 49 +++++++++++++++++++ .../repositories/uri/URLRepository.java | 0 settings.gradle | 1 + 8 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 modules/url-repository/build.gradle rename {core => modules/url-repository}/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java (100%) rename {core => modules/url-repository}/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java (100%) create mode 100644 modules/url-repository/src/main/java/org/elasticsearch/repositories/URLRespositoryPlugin.java rename {core => modules/url-repository}/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java (100%) diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 34a574077b2d9..c26a04aafc89c 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -54,9 +54,9 @@ import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.discovery.zen.ElectMasterService; -import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.discovery.zen.FaultDetection; import org.elasticsearch.discovery.zen.UnicastZenPing; +import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.gateway.GatewayService; @@ -80,7 +80,6 @@ import org.elasticsearch.node.Node; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.fs.FsRepository; -import org.elasticsearch.repositories.uri.URLRepository; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; @@ -343,9 +342,6 @@ public void apply(Settings value, Settings current, Settings previous) { Node.NODE_INGEST_SETTING, Node.NODE_ATTRIBUTES, Node.NODE_LOCAL_STORAGE_SETTING, - URLRepository.ALLOWED_URLS_SETTING, - URLRepository.REPOSITORIES_URL_SETTING, - URLRepository.SUPPORTED_PROTOCOLS_SETTING, TransportMasterNodeReadAction.FORCE_LOCAL_SETTING, AutoCreateIndex.AUTO_CREATE_INDEX_SETTING, BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX, diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java index d03e2c1ac349f..3073e1f0b0ef6 100644 --- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java +++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java @@ -19,24 +19,22 @@ package org.elasticsearch.repositories; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.elasticsearch.common.inject.AbstractModule; -import org.elasticsearch.common.inject.binder.LinkedBindingBuilder; import org.elasticsearch.common.inject.multibindings.MapBinder; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.fs.FsRepository; -import org.elasticsearch.repositories.uri.URLRepository; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.SnapshotShardsService; import org.elasticsearch.snapshots.SnapshotsService; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * Sets up classes for Snapshot/Restore. */ @@ -47,7 +45,6 @@ public class RepositoriesModule extends AbstractModule { public RepositoriesModule(Environment env, List repoPlugins, NamedXContentRegistry namedXContentRegistry) { Map factories = new HashMap<>(); factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env, namedXContentRegistry)); - factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env, namedXContentRegistry)); for (RepositoryPlugin repoPlugin : repoPlugins) { Map newRepoTypes = repoPlugin.getRepositories(env, namedXContentRegistry); diff --git a/modules/url-repository/build.gradle b/modules/url-repository/build.gradle new file mode 100644 index 0000000000000..c8dcd38b6fdb4 --- /dev/null +++ b/modules/url-repository/build.gradle @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' + classname 'org.elasticsearch.ingest.common.IngestCommonPlugin' +} + +dependencies { +} + +compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" +compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" + +thirdPartyAudit.excludes = [ + +] \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/modules/url-repository/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java similarity index 100% rename from core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java rename to modules/url-repository/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/url-repository/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java similarity index 100% rename from core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java rename to modules/url-repository/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java diff --git a/modules/url-repository/src/main/java/org/elasticsearch/repositories/URLRespositoryPlugin.java b/modules/url-repository/src/main/java/org/elasticsearch/repositories/URLRespositoryPlugin.java new file mode 100644 index 0000000000000..3ddd6d59c2776 --- /dev/null +++ b/modules/url-repository/src/main/java/org/elasticsearch/repositories/URLRespositoryPlugin.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.RepositoryPlugin; +import org.elasticsearch.repositories.uri.URLRepository; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class URLRespositoryPlugin extends Plugin implements RepositoryPlugin { + + @Override + public List> getSettings() { + return Arrays.asList( + URLRepository.ALLOWED_URLS_SETTING, + URLRepository.REPOSITORIES_URL_SETTING, + URLRepository.SUPPORTED_PROTOCOLS_SETTING + ); + } + + @Override + public Map getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) { + return Collections.singletonMap(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env, namedXContentRegistry)); + } +} diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/modules/url-repository/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java similarity index 100% rename from core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java rename to modules/url-repository/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java diff --git a/settings.gradle b/settings.gradle index 1125e84325c42..e4f880ce3109c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -31,6 +31,7 @@ List projects = [ 'modules:transport-netty4', 'modules:reindex', 'modules:percolator', + 'modules:url-repository', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', 'plugins:analysis-phonetic', From 563b616abb51ef2d363b818db1343cc918710cd5 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 18 Jan 2017 14:15:01 -0600 Subject: [PATCH 02/24] Fix compile issues --- .../bwcompat/RestoreBackwardsCompatIT.java | 30 +++++++++---------- modules/url-repository/build.gradle | 4 +-- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index 6d13d3cf418c4..21608da6551a5 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; -import org.elasticsearch.repositories.uri.URLRepository; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.snapshots.RestoreInfo; @@ -40,7 +39,6 @@ import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -62,25 +60,25 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - if (randomBoolean()) { +// if (randomBoolean()) { // Configure using path.repo return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(Environment.PATH_REPO_SETTING.getKey(), getBwcIndicesPath()) .build(); - } else { - // Configure using url white list - try { - URI repoJarPatternUri = new URI("jar:" + getBwcIndicesPath().toUri().toString() + "*.zip!/repo/"); - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .putArray(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoJarPatternUri.toString()) - .build(); - } catch (URISyntaxException ex) { - throw new IllegalArgumentException(ex); - } - - } +// } else { +// // Configure using url white list +// try { +// URI repoJarPatternUri = new URI("jar:" + getBwcIndicesPath().toUri().toString() + "*.zip!/repo/"); +// return Settings.builder() +// .put(super.nodeSettings(nodeOrdinal)) +// .putArray(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoJarPatternUri.toString()) +// .build(); +// } catch (URISyntaxException ex) { +// throw new IllegalArgumentException(ex); +// } +// +// } } public void testRestoreOldSnapshots() throws Exception { diff --git a/modules/url-repository/build.gradle b/modules/url-repository/build.gradle index c8dcd38b6fdb4..f8e16927d2c16 100644 --- a/modules/url-repository/build.gradle +++ b/modules/url-repository/build.gradle @@ -19,7 +19,7 @@ esplugin { description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' - classname 'org.elasticsearch.ingest.common.IngestCommonPlugin' + classname 'org.elasticsearch.repositories.URLRespositoryPlugin' } dependencies { @@ -30,4 +30,4 @@ compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" thirdPartyAudit.excludes = [ -] \ No newline at end of file +] From 311d8905472e6e4b8538e7b62155a70cbb52f636 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 19 Jan 2017 14:31:33 -0600 Subject: [PATCH 03/24] Work on restructuring module --- modules/{url-repository => repositories}/build.gradle | 0 .../elasticsearch/common/blobstore/url/URLBlobContainer.java | 0 .../org/elasticsearch/common/blobstore/url/URLBlobStore.java | 0 .../elasticsearch/plugin/repositories/RepositoriesPlugin.java} | 2 +- .../java/org/elasticsearch/repositories/url}/URLRepository.java | 0 5 files changed, 1 insertion(+), 1 deletion(-) rename modules/{url-repository => repositories}/build.gradle (100%) rename modules/{url-repository => repositories}/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java (100%) rename modules/{url-repository => repositories}/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java (100%) rename modules/{url-repository/src/main/java/org/elasticsearch/repositories/URLRespositoryPlugin.java => repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java} (95%) rename modules/{url-repository/src/main/java/org/elasticsearch/repositories/uri => repositories/src/main/java/org/elasticsearch/repositories/url}/URLRepository.java (100%) diff --git a/modules/url-repository/build.gradle b/modules/repositories/build.gradle similarity index 100% rename from modules/url-repository/build.gradle rename to modules/repositories/build.gradle diff --git a/modules/url-repository/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java similarity index 100% rename from modules/url-repository/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java rename to modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java diff --git a/modules/url-repository/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java similarity index 100% rename from modules/url-repository/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java rename to modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java diff --git a/modules/url-repository/src/main/java/org/elasticsearch/repositories/URLRespositoryPlugin.java b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java similarity index 95% rename from modules/url-repository/src/main/java/org/elasticsearch/repositories/URLRespositoryPlugin.java rename to modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java index 3ddd6d59c2776..83d258ce8fc49 100644 --- a/modules/url-repository/src/main/java/org/elasticsearch/repositories/URLRespositoryPlugin.java +++ b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java @@ -31,7 +31,7 @@ import java.util.List; import java.util.Map; -public class URLRespositoryPlugin extends Plugin implements RepositoryPlugin { +public class RespositoriesPlugin extends Plugin implements RepositoryPlugin { @Override public List> getSettings() { diff --git a/modules/url-repository/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java similarity index 100% rename from modules/url-repository/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java rename to modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java From 669c4efb417144c1a0758ea3f456c395f1578e62 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 19 Jan 2017 14:31:38 -0600 Subject: [PATCH 04/24] Work on restructuring module --- modules/repositories/build.gradle | 4 ++-- .../plugin/repositories/RepositoriesPlugin.java | 7 ++++--- .../org/elasticsearch/repositories/url/URLRepository.java | 2 +- settings.gradle | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/repositories/build.gradle b/modules/repositories/build.gradle index f8e16927d2c16..bc439cccb6d57 100644 --- a/modules/repositories/build.gradle +++ b/modules/repositories/build.gradle @@ -18,8 +18,8 @@ */ esplugin { - description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' - classname 'org.elasticsearch.repositories.URLRespositoryPlugin' + description 'Module for generic repositories' + classname 'org.elasticsearch.repositories.RepositoriesPlugin' } dependencies { diff --git a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java index 83d258ce8fc49..237a298d52b47 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java +++ b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java @@ -17,21 +17,22 @@ * under the License. */ -package org.elasticsearch.repositories; +package org.elasticsearch.plugin.repositories; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.repositories.uri.URLRepository; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.url.URLRepository; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -public class RespositoriesPlugin extends Plugin implements RepositoryPlugin { +public class RepositoriesPlugin extends Plugin implements RepositoryPlugin { @Override public List> getSettings() { diff --git a/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index fdeb27819bf08..bdb7df2e610e4 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java +++ b/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.repositories.uri; +package org.elasticsearch.repositories.url; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobPath; diff --git a/settings.gradle b/settings.gradle index 617a83d4cb14e..6c155a9a6e5fa 100644 --- a/settings.gradle +++ b/settings.gradle @@ -31,7 +31,7 @@ List projects = [ 'modules:transport-netty4', 'modules:reindex', 'modules:percolator', - 'modules:url-repository', + 'modules:repositories', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', 'plugins:analysis-phonetic', From 18fefbdc5f0d608548da6632bbfd0ce31ac47f80 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 19 Jan 2017 20:30:01 -0600 Subject: [PATCH 05/24] WIP: work on implementing test for urlblobstore --- modules/repositories/build.gradle | 2 +- .../blobstore}/url/URLRepository.java | 3 +- .../repositories/RepositoriesPlugin.java | 2 +- .../url/URLBlockStoreContainerTests.java | 75 +++++++++++++++++++ 4 files changed, 78 insertions(+), 4 deletions(-) rename modules/repositories/src/main/java/org/elasticsearch/{repositories => common/blobstore}/url/URLRepository.java (98%) create mode 100644 modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlockStoreContainerTests.java diff --git a/modules/repositories/build.gradle b/modules/repositories/build.gradle index bc439cccb6d57..eac5a961fd20f 100644 --- a/modules/repositories/build.gradle +++ b/modules/repositories/build.gradle @@ -19,7 +19,7 @@ esplugin { description 'Module for generic repositories' - classname 'org.elasticsearch.repositories.RepositoriesPlugin' + classname 'org.elasticsearch.plugin.repositories.RepositoriesPlugin' } dependencies { diff --git a/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLRepository.java similarity index 98% rename from modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java rename to modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLRepository.java index bdb7df2e610e4..a8ccbd294fa69 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java +++ b/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLRepository.java @@ -17,12 +17,11 @@ * under the License. */ -package org.elasticsearch.repositories.url; +package org.elasticsearch.common.blobstore.url; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.blobstore.url.URLBlobStore; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.URIPattern; diff --git a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java index 237a298d52b47..dff5983041220 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java +++ b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java @@ -25,7 +25,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.Repository; -import org.elasticsearch.repositories.url.URLRepository; +import org.elasticsearch.common.blobstore.url.URLRepository; import java.util.Arrays; import java.util.Collections; diff --git a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlockStoreContainerTests.java b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlockStoreContainerTests.java new file mode 100644 index 0000000000000..46cd2ab41dacb --- /dev/null +++ b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlockStoreContainerTests.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.blobstore.url; + +import com.sun.net.httpserver.HttpServer; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.test.ESTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URL; + +public class URLBlockStoreContainerTests extends ESTestCase { + + private static HttpServer httpServer; + private static byte[] message = new byte[1024]; + + @BeforeClass + public static void startHttp() throws Exception { +// logDir = createTempDir(); + httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 6001), 0); + + httpServer.createContext("/hjdfd", (s) -> { + s.sendResponseHeaders(200, message.length); + OutputStream responseBody = s.getResponseBody(); + responseBody.write(message); + responseBody.close(); + }); + + httpServer.start(); + } + + @AfterClass + public static void stopHttp() throws IOException { + httpServer.stop(0); + httpServer = null; + } + + public void testBlobStore() throws IOException { + Settings settings = Settings.EMPTY; + URLBlobStore urlBlobStore = new URLBlobStore(settings, new URL("http://localhost:6001")); + BlobContainer contain = urlBlobStore.blobContainer(BlobPath.cleanPath()); + InputStream stream = contain.readBlob("/hjdfd"); + try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(stream))) { + System.out.println(bufferedReader.readLine()); + } + } +} From 4e05efc95e70e7917e0193695f2f7cc25929d0fe Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Sat, 21 Jan 2017 21:05:50 -0600 Subject: [PATCH 06/24] Implement basic blogstore tests --- ...ainerTests.java => URLBlobStoreTests.java} | 48 ++++++++++++++----- .../blobstore/url/URLRepositoryTests.java | 41 ++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) rename modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/{URLBlockStoreContainerTests.java => URLBlobStoreTests.java} (55%) create mode 100644 modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java diff --git a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlockStoreContainerTests.java b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java similarity index 55% rename from modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlockStoreContainerTests.java rename to modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java index 46cd2ab41dacb..61cc5c0273d75 100644 --- a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlockStoreContainerTests.java +++ b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java @@ -26,28 +26,35 @@ import org.elasticsearch.mocksocket.MockHttpServer; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.NoSuchFileException; -public class URLBlockStoreContainerTests extends ESTestCase { +public class URLBlobStoreTests extends ESTestCase { private static HttpServer httpServer; - private static byte[] message = new byte[1024]; + private static String blobName; + private static byte[] message = new byte[512]; + private URLBlobStore urlBlobStore; @BeforeClass public static void startHttp() throws Exception { -// logDir = createTempDir(); + for (int i = 0; i < message.length; ++i) { + message[i] = randomByte(); + } + blobName = randomAsciiOfLength(8); + httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 6001), 0); - httpServer.createContext("/hjdfd", (s) -> { + httpServer.createContext("/indices/" + blobName, (s) -> { s.sendResponseHeaders(200, message.length); OutputStream responseBody = s.getResponseBody(); responseBody.write(message); @@ -63,13 +70,30 @@ public static void stopHttp() throws IOException { httpServer = null; } - public void testBlobStore() throws IOException { + @Before + public void storeSetup() throws MalformedURLException { Settings settings = Settings.EMPTY; - URLBlobStore urlBlobStore = new URLBlobStore(settings, new URL("http://localhost:6001")); - BlobContainer contain = urlBlobStore.blobContainer(BlobPath.cleanPath()); - InputStream stream = contain.readBlob("/hjdfd"); - try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(stream))) { - System.out.println(bufferedReader.readLine()); + String spec = "http://localhost:6001/"; + urlBlobStore = new URLBlobStore(settings, new URL(spec)); + } + + public void testURLBlobStoreCanReadBlob() throws IOException { + BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices")); + try (InputStream stream = container.readBlob(blobName)) { + byte[] bytes = new byte[message.length]; + int read = stream.read(bytes); + assertEquals(message.length, read); + assertArrayEquals(message, bytes); + } + } + + public void testNoBlobFound() throws IOException { + BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices")); + String incorrectBlobName = "incorrect_" + blobName; + try (InputStream stream = container.readBlob(incorrectBlobName)) { + fail("Should have thrown NoSuchFileException exception"); + } catch (NoSuchFileException e) { + assertEquals(String.format("[%s] blob not found", incorrectBlobName), e.getMessage()); } } } diff --git a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java new file mode 100644 index 0000000000000..4a47c25eac748 --- /dev/null +++ b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.blobstore.url; + +import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Collections; + +public class URLRepositoryTests extends ESTestCase { + + public void testThing() throws IOException { +// RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, Settings.EMPTY); +// Environment environment = new Environment(Settings.EMPTY); +// URLRepository urlRepository = new URLRepository(repositoryMetaData, environment, new NamedXContentRegistry(Collections.emptyList())); +// System.out.println(urlRepository); + } + + +} From 7bf6f4fb9df53b1707a7eb17c4007ce32de6e90d Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Sun, 22 Jan 2017 12:46:10 -0600 Subject: [PATCH 07/24] Add URLRepository tests --- .../blobstore/url/URLRepositoryTests.java | 49 +++++++++++++++++-- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java index 4a47c25eac748..8adece76f67c1 100644 --- a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java +++ b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java @@ -23,19 +23,58 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; +import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.nio.file.Path; import java.util.Collections; public class URLRepositoryTests extends ESTestCase { - public void testThing() throws IOException { -// RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, Settings.EMPTY); -// Environment environment = new Environment(Settings.EMPTY); -// URLRepository urlRepository = new URLRepository(repositoryMetaData, environment, new NamedXContentRegistry(Collections.emptyList())); -// System.out.println(urlRepository); + public void testWhiteListingRepoURL() throws IOException { + String repoPath = createTempDir().resolve("repository").toUri().toURL().toString(); + System.out.println(repoPath); + Settings baseSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoPath) + .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath) + .build(); + RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings); + new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); } + public void testIfNotWhiteListedMustSetRepoURL() throws IOException { + String repoPath = createTempDir().resolve("repository").toUri().toURL().toString(); + Settings baseSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath) + .build(); + RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings); + try { + new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); + fail("RepositoryException should have been thrown."); + } catch (RepositoryException e) { + assertEquals(String.format("[url] file url [%s] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls", repoPath), e.getMessage()); + } + } + + public void testMustBeSupportedProtocol() throws IOException { + Path directory = createTempDir(); + String repoPath = directory.resolve("repository").toUri().toURL().toString(); + Settings baseSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(Environment.PATH_REPO_SETTING.getKey(), directory.toString()) + .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath) + .put(URLRepository.SUPPORTED_PROTOCOLS_SETTING.getKey(), "http,https") + .build(); + RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings); + try { + new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); + fail("RepositoryException should have been thrown."); + } catch (RepositoryException e) { + assertEquals(String.format("[url] unsupported url protocol [file] from URL [%s]", repoPath), e.getMessage()); + } + } } From b263edcfe63ee031ec60faf72ce61015937b2415 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jan 2017 14:44:40 -0600 Subject: [PATCH 08/24] Fix restore bw compat test --- .../bwcompat/RestoreBackwardsCompatIT.java | 51 +++++++++---------- .../blobstore/url/URLBlobStoreTests.java | 3 +- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index 408bef1b5e615..4955674eb1557 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -26,19 +26,22 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; +import org.elasticsearch.common.io.FileTestUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotRestoreException; +import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.VersionUtils; +import org.junit.BeforeClass; import java.io.IOException; -import java.net.URI; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -59,27 +62,19 @@ @ClusterScope(scope = Scope.TEST) public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { + private static Path repoPath; + @Override protected Settings nodeSettings(int nodeOrdinal) { -// if (randomBoolean()) { - // Configure using path.repo - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(Environment.PATH_REPO_SETTING.getKey(), getBwcIndicesPath()) - .build(); -// } else { -// // Configure using url white list -// try { -// URI repoJarPatternUri = new URI("jar:" + getBwcIndicesPath().toUri().toString() + "*.zip!/repo/"); -// return Settings.builder() -// .put(super.nodeSettings(nodeOrdinal)) -// .putArray(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoJarPatternUri.toString()) -// .build(); -// } catch (URISyntaxException ex) { -// throw new IllegalArgumentException(ex); -// } -// -// } + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(Environment.PATH_REPO_SETTING.getKey(), repoPath) + .build(); + } + + @BeforeClass + public static void repoSetup() throws IOException { + repoPath = createTempDir("repositories"); } public void testRestoreOldSnapshots() throws Exception { @@ -96,7 +91,8 @@ public void testRestoreOldSnapshots() throws Exception { for (Version v : VersionUtils.allReleasedVersions()) { if (VersionUtils.isSnapshot(v)) continue; // snapshots are unreleased, so there is no backcompat yet if (v.isRelease() == false) continue; // no guarantees for prereleases - if (v.before(Version.CURRENT.minimumIndexCompatibilityVersion())) continue; // we only support versions N and N-1 + if (v.before(Version.CURRENT.minimumIndexCompatibilityVersion())) + continue; // we only support versions N and N-1 if (v.equals(Version.CURRENT)) continue; // the current version is always compatible with itself expectedVersions.add(v.toString()); } @@ -149,13 +145,14 @@ private List listRepoVersions(String prefix) throws Exception { } private void createRepo(String prefix, String version, String repo) throws Exception { - Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip"); - URI repoFileUri = repoFile.toUri(); - URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/"); + Path repoFileFromBuild = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip"); + String repoFileName = repoFileFromBuild.getFileName().toString().split(".zip")[0]; + Path fsRepoPath = repoPath.resolve(repoFileName); + FileTestUtils.unzip(repoFileFromBuild, fsRepoPath, null); logger.info("--> creating repository [{}] for version [{}]", repo, version); assertAcked(client().admin().cluster().preparePutRepository(repo) - .setType("url").setSettings(Settings.builder() - .put("url", repoJarUri.toString()))); + .setType(MockRepository.TYPE).setSettings(Settings.builder() + .put(FsRepository.REPOSITORIES_LOCATION_SETTING.getKey(), fsRepoPath.getParent().relativize(fsRepoPath).resolve("repo").toString()))); } private void testOldSnapshot(String version, String repo, String snapshot) throws IOException { @@ -196,7 +193,7 @@ private void testOldSnapshot(String version, String repo, String snapshot) throw equalTo("{\"type1\":{\"_source\":{\"enabled\":0}}}"), equalTo("{\"type1\":{\"_source\":{\"enabled\":\"off\"}}}"), equalTo("{\"type1\":{\"_source\":{\"enabled\":\"no\"}}}") - )); + )); assertThat(template.aliases().size(), equalTo(3)); assertThat(template.aliases().get("alias1"), notNullValue()); assertThat(template.aliases().get("alias2").filter().string(), containsString(version)); diff --git a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java index 61cc5c0273d75..cb78036a642aa 100644 --- a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java +++ b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java @@ -90,8 +90,9 @@ public void testURLBlobStoreCanReadBlob() throws IOException { public void testNoBlobFound() throws IOException { BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices")); String incorrectBlobName = "incorrect_" + blobName; - try (InputStream stream = container.readBlob(incorrectBlobName)) { + try (InputStream ignored = container.readBlob(incorrectBlobName)) { fail("Should have thrown NoSuchFileException exception"); + ignored.read(); } catch (NoSuchFileException e) { assertEquals(String.format("[%s] blob not found", incorrectBlobName), e.getMessage()); } From 6c574e2f5156392b5ea794c3b0fdf2f82d7908cc Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jan 2017 14:55:36 -0600 Subject: [PATCH 09/24] Reorganize packages --- .../elasticsearch/plugin/repositories/RepositoriesPlugin.java | 2 +- .../{common/blobstore => repositories}/url/URLRepository.java | 3 ++- .../blobstore => repositories}/url/URLRepositoryTests.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) rename modules/repositories/src/main/java/org/elasticsearch/{common/blobstore => repositories}/url/URLRepository.java (98%) rename modules/repositories/src/test/java/org/elasticsearch/{common/blobstore => repositories}/url/URLRepositoryTests.java (97%) diff --git a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java index dff5983041220..237a298d52b47 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java +++ b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java @@ -25,7 +25,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.Repository; -import org.elasticsearch.common.blobstore.url.URLRepository; +import org.elasticsearch.repositories.url.URLRepository; import java.util.Arrays; import java.util.Collections; diff --git a/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLRepository.java b/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java similarity index 98% rename from modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLRepository.java rename to modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index a8ccbd294fa69..bdb7df2e610e4 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLRepository.java +++ b/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -17,11 +17,12 @@ * under the License. */ -package org.elasticsearch.common.blobstore.url; +package org.elasticsearch.repositories.url; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.url.URLBlobStore; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.URIPattern; diff --git a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java b/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java similarity index 97% rename from modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java rename to modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java index 8adece76f67c1..70ed9575ef3dc 100644 --- a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLRepositoryTests.java +++ b/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -17,13 +17,14 @@ * under the License. */ -package org.elasticsearch.common.blobstore.url; +package org.elasticsearch.repositories.url; import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.url.URLRepository; import org.elasticsearch.test.ESTestCase; import java.io.IOException; From 6ecf433ce971378e3a2505fb84e1d441776ce2e1 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jan 2017 15:42:26 -0600 Subject: [PATCH 10/24] Fix checkstyles and modify for review feedback --- buildSrc/src/main/resources/checkstyle_suppressions.xml | 2 -- .../elasticsearch/bwcompat/RestoreBackwardsCompatIT.java | 3 +-- modules/repositories/build.gradle | 6 ------ .../elasticsearch/common/blobstore/url/URLBlobStore.java | 3 ++- .../plugin/repositories/RepositoriesPlugin.java | 2 +- .../org/elasticsearch/repositories/url/URLRepository.java | 8 ++++++-- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 61e9a205d3d2d..eb64cf180e994 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -257,7 +257,6 @@ - @@ -438,7 +437,6 @@ - diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index 4955674eb1557..9dd9e9dd4a62d 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -91,8 +91,7 @@ public void testRestoreOldSnapshots() throws Exception { for (Version v : VersionUtils.allReleasedVersions()) { if (VersionUtils.isSnapshot(v)) continue; // snapshots are unreleased, so there is no backcompat yet if (v.isRelease() == false) continue; // no guarantees for prereleases - if (v.before(Version.CURRENT.minimumIndexCompatibilityVersion())) - continue; // we only support versions N and N-1 + if (v.before(Version.CURRENT.minimumIndexCompatibilityVersion())) continue; // we only support versions N and N-1 if (v.equals(Version.CURRENT)) continue; // the current version is always compatible with itself expectedVersions.add(v.toString()); } diff --git a/modules/repositories/build.gradle b/modules/repositories/build.gradle index eac5a961fd20f..db0165fb25624 100644 --- a/modules/repositories/build.gradle +++ b/modules/repositories/build.gradle @@ -22,12 +22,6 @@ esplugin { classname 'org.elasticsearch.plugin.repositories.RepositoriesPlugin' } -dependencies { -} - compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" -thirdPartyAudit.excludes = [ - -] diff --git a/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java index 2386f79c076f1..158ecff9b2b4e 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java +++ b/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java @@ -55,7 +55,8 @@ public class URLBlobStore extends AbstractComponent implements BlobStore { public URLBlobStore(Settings settings, URL path) { super(settings); this.path = path; - this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); + this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", + new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); } /** diff --git a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java index 237a298d52b47..49db847e8447b 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java +++ b/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java @@ -45,6 +45,6 @@ public List> getSettings() { @Override public Map getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) { - return Collections.singletonMap(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env, namedXContentRegistry)); + return Collections.singletonMap(URLRepository.TYPE, metadata -> new URLRepository(metadata, env, namedXContentRegistry)); } } diff --git a/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index bdb7df2e610e4..c1128fd683a70 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java +++ b/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -127,8 +127,12 @@ private URL checkURL(URL url) { // We didn't match white list - try to resolve against path.repo URL normalizedUrl = environment.resolveRepoURL(url); if (normalizedUrl == null) { - logger.warn("The specified url [{}] doesn't start with any repository paths specified by the path.repo setting or by {} setting: [{}] ", url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles()); - throw new RepositoryException(getMetadata().name(), "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + ALLOWED_URLS_SETTING.getKey()); + String logMessage = "The specified url [{}] doesn't start with any repository paths specified by the " + + "path.repo setting or by {} setting: [{}] "; + logger.warn(logMessage, url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles()); + String exceptionMessage = "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + + ALLOWED_URLS_SETTING.getKey(); + throw new RepositoryException(getMetadata().name(), exceptionMessage); } return normalizedUrl; } From ee80d3ba517fa78faf833e8367eb4cf6c3d3f8ab Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jan 2017 16:13:44 -0600 Subject: [PATCH 11/24] Fix line length --- .../org/elasticsearch/repositories/url/URLRepositoryTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java index 70ed9575ef3dc..c9fadfc839111 100644 --- a/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java +++ b/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -56,7 +56,8 @@ public void testIfNotWhiteListedMustSetRepoURL() throws IOException { new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); fail("RepositoryException should have been thrown."); } catch (RepositoryException e) { - assertEquals(String.format("[url] file url [%s] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls", repoPath), e.getMessage()); + String msg = "[url] file url [%s] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls"; + assertEquals(String.format(msg, repoPath), e.getMessage()); } } From ec750a9e2604c4d91a4835ab4b6ba3d4760ce8e4 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jan 2017 17:07:46 -0600 Subject: [PATCH 12/24] Remove system.out call --- .../org/elasticsearch/repositories/url/URLRepositoryTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java index c9fadfc839111..18b680cb0718e 100644 --- a/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java +++ b/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -35,7 +35,6 @@ public class URLRepositoryTests extends ESTestCase { public void testWhiteListingRepoURL() throws IOException { String repoPath = createTempDir().resolve("repository").toUri().toURL().toString(); - System.out.println(repoPath); Settings baseSettings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoPath) From 6f661c8443b7518afc4bef88de76647a4796a29a Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 23 Jan 2017 17:27:13 -0600 Subject: [PATCH 13/24] Fix forbidden issues --- .../common/blobstore/url/URLBlobStoreTests.java | 2 ++ .../repositories/url/URLRepositoryTests.java | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java index cb78036a642aa..4df83c58bd38b 100644 --- a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java +++ b/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.blobstore.url; import com.sun.net.httpserver.HttpServer; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.settings.Settings; @@ -38,6 +39,7 @@ import java.net.URL; import java.nio.file.NoSuchFileException; +@SuppressForbidden(reason = "use http server") public class URLBlobStoreTests extends ESTestCase { private static HttpServer httpServer; diff --git a/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java index 18b680cb0718e..ea274eeae602a 100644 --- a/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java +++ b/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.repositories.RepositoryException; -import org.elasticsearch.repositories.url.URLRepository; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -55,8 +54,9 @@ public void testIfNotWhiteListedMustSetRepoURL() throws IOException { new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); fail("RepositoryException should have been thrown."); } catch (RepositoryException e) { - String msg = "[url] file url [%s] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls"; - assertEquals(String.format(msg, repoPath), e.getMessage()); + String msg = "[url] file url [" + repoPath + + "] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls"; + assertEquals(msg, e.getMessage()); } } @@ -74,7 +74,7 @@ public void testMustBeSupportedProtocol() throws IOException { new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList())); fail("RepositoryException should have been thrown."); } catch (RepositoryException e) { - assertEquals(String.format("[url] unsupported url protocol [file] from URL [%s]", repoPath), e.getMessage()); + assertEquals("[url] unsupported url protocol [file] from URL [" + repoPath +"]", e.getMessage()); } } From fa40f2c38d16086df710c7e38a80b753c1f207a0 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 09:38:14 -0600 Subject: [PATCH 14/24] Rename url module repository --- modules/{repositories => repository-url}/build.gradle | 0 .../elasticsearch/common/blobstore/url/URLBlobContainer.java | 0 .../org/elasticsearch/common/blobstore/url/URLBlobStore.java | 0 .../plugin/repository/url/URLRepositoryPlugin.java} | 4 ++-- .../org/elasticsearch/repositories/url/URLRepository.java | 0 .../elasticsearch/common/blobstore/url/URLBlobStoreTests.java | 0 .../elasticsearch/repositories/url/URLRepositoryTests.java | 0 settings.gradle | 2 +- 8 files changed, 3 insertions(+), 3 deletions(-) rename modules/{repositories => repository-url}/build.gradle (100%) rename modules/{repositories => repository-url}/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java (100%) rename modules/{repositories => repository-url}/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java (100%) rename modules/{repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java => repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java} (93%) rename modules/{repositories => repository-url}/src/main/java/org/elasticsearch/repositories/url/URLRepository.java (100%) rename modules/{repositories => repository-url}/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java (100%) rename modules/{repositories => repository-url}/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java (100%) diff --git a/modules/repositories/build.gradle b/modules/repository-url/build.gradle similarity index 100% rename from modules/repositories/build.gradle rename to modules/repository-url/build.gradle diff --git a/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java similarity index 100% rename from modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java rename to modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java diff --git a/modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java similarity index 100% rename from modules/repositories/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java rename to modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java diff --git a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java similarity index 93% rename from modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java rename to modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java index 49db847e8447b..a28413d213a97 100644 --- a/modules/repositories/src/main/java/org/elasticsearch/plugin/repositories/RepositoriesPlugin.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.plugin.repositories; +package org.elasticsearch.plugin.repository.url; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -32,7 +32,7 @@ import java.util.List; import java.util.Map; -public class RepositoriesPlugin extends Plugin implements RepositoryPlugin { +public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin { @Override public List> getSettings() { diff --git a/modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java similarity index 100% rename from modules/repositories/src/main/java/org/elasticsearch/repositories/url/URLRepository.java rename to modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java diff --git a/modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java similarity index 100% rename from modules/repositories/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java rename to modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java diff --git a/modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java similarity index 100% rename from modules/repositories/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java rename to modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java diff --git a/settings.gradle b/settings.gradle index 6c155a9a6e5fa..5b6ff5d67b8c8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -31,7 +31,7 @@ List projects = [ 'modules:transport-netty4', 'modules:reindex', 'modules:percolator', - 'modules:repositories', + 'modules:repository-url', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', 'plugins:analysis-phonetic', From b8b6132dc6f8029823ae1c8899cfb4c33e2549a2 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 09:47:55 -0600 Subject: [PATCH 15/24] Remove dupliated tests --- .../snapshots/RepositoriesIT.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java index 639b60d6d09ff..0f6dcec818fc9 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -142,30 +142,6 @@ public void testMisconfiguredRepository() throws Exception { } catch (RepositoryException ex) { assertThat(ex.toString(), containsString("location [" + location + "] doesn't match any of the locations specified by path.repo")); } - - String repoUrl = invalidRepoPath.toAbsolutePath().toUri().toURL().toString(); - String unsupportedUrl = repoUrl.replace("file:/", "netdoc:/"); - logger.info("--> trying creating url repository with unsupported url protocol"); - try { - client().admin().cluster().preparePutRepository("test-repo") - .setType("url").setSettings(Settings.builder().put("url", unsupportedUrl)) - .get(); - fail("Shouldn't be here"); - } catch (RepositoryException ex) { - assertThat(ex.toString(), - either(containsString("unsupported url protocol [netdoc]")) - .or(containsString("unknown protocol: netdoc"))); // newer versions of JDK 9 - } - - logger.info("--> trying creating url repository with location that is not registered in path.repo setting"); - try { - client().admin().cluster().preparePutRepository("test-repo") - .setType("url").setSettings(Settings.builder().put("url", invalidRepoPath.toUri().toURL())) - .get(); - fail("Shouldn't be here"); - } catch (RepositoryException ex) { - assertThat(ex.toString(), containsString("doesn't match any of the locations specified by path.repo")); - } } public void testRepositoryAckTimeout() throws Exception { From 344eeb248e5a51e891cf574d279f17f3367d8e9f Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 11:29:04 -0600 Subject: [PATCH 16/24] Move integration test --- .../SharedClusterSnapshotRestoreIT.java | 57 --------- .../url/URLSnapshotRestoreIT.java | 112 ++++++++++++++++++ 2 files changed, 112 insertions(+), 57 deletions(-) create mode 100644 modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 9a4a302ef975a..308f79393a36e 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1442,63 +1442,6 @@ public void testDeleteRepositoryWhileSnapshotting() throws Exception { assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); } - public void testUrlRepository() throws Exception { - Client client = client(); - - logger.info("--> creating repository"); - Path repositoryLocation = randomRepoPath(); - assertAcked(client.admin().cluster().preparePutRepository("test-repo") - .setType("fs").setSettings(Settings.builder() - .put("location", repositoryLocation) - .put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); - - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); - - logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); - - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); - - logger.info("--> delete index"); - cluster().wipeIndices("test-idx"); - - logger.info("--> create read-only URL repository"); - assertAcked(client.admin().cluster().preparePutRepository("url-repo") - .setType("url").setSettings(Settings.builder() - .put("url", repositoryLocation.toUri().toURL()) - .put("list_directories", randomBoolean()))); - logger.info("--> restore index after deletion"); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet(); - assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); - - logger.info("--> list available shapshots"); - GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); - assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); - - logger.info("--> delete snapshot"); - DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get(); - assertAcked(deleteSnapshotResponse); - - logger.info("--> list available shapshot again, no snapshots should be returned"); - getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); - assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0)); - } - @TestLogging("_root:DEBUG") // this fails every now and then: https://github.com/elastic/elasticsearch/issues/18121 but without // more logs we cannot find out why public void testReadonlyRepository() throws Exception { diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java new file mode 100644 index 0000000000000..ad5cdfc4f0224 --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.url; + +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.plugin.repository.url.URLRepositoryPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Before; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) +public class URLSnapshotRestoreIT extends ESIntegTestCase { + + protected boolean ignoreExternalCluster() { + return true; + } + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(URLRepositoryPlugin.class); + } + + public void testUrlRepository() throws Exception { + Client client = client(); + + logger.info("--> creating repository"); + Path repositoryLocation = randomRepoPath(); + assertAcked(client.admin().cluster().preparePutRepository("test-repo") + .setType("fs").setSettings(Settings.builder() + .put("location", repositoryLocation) + .put("compress", randomBoolean()) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + + createIndex("test-idx"); + ensureGreen(); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); + } + refresh(); + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); + + logger.info("--> snapshot"); + CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); + + assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); + + logger.info("--> delete index"); + cluster().wipeIndices("test-idx"); + + logger.info("--> create read-only URL repository"); + assertAcked(client.admin().cluster().preparePutRepository("url-repo") + .setType(URLRepository.TYPE).setSettings(Settings.builder() + .put("url", repositoryLocation.toUri().toURL()) + .put("list_directories", randomBoolean()))); + logger.info("--> restore index after deletion"); + RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet(); + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); + + logger.info("--> list available shapshots"); + GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); + assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); + + logger.info("--> delete snapshot"); + DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get(); + assertAcked(deleteSnapshotResponse); + + logger.info("--> list available shapshot again, no snapshots should be returned"); + getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get(); + assertThat(getSnapshotsResponse.getSnapshots(), notNullValue()); + assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0)); + } +} From 937ce941950f67728994f64edff01b6d8eb47692 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 13:07:19 -0600 Subject: [PATCH 17/24] Fix checkstyle errors --- .../url/URLSnapshotRestoreIT.java | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java index ad5cdfc4f0224..74a0e76b572f9 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ESIntegTestCase; -import org.junit.Before; import java.nio.file.Path; import java.util.Collection; @@ -75,11 +74,27 @@ public void testUrlRepository() throws Exception { assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get(); + CreateSnapshotResponse createSnapshotResponse = client + .admin() + .cluster() + .prepareCreateSnapshot("test-repo", "test-snap") + .setWaitForCompletion(true) + .setIndices("test-idx") + .get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); - assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); - - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); + int actualTotalShards = createSnapshotResponse.getSnapshotInfo().totalShards(); + assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(actualTotalShards)); + + SnapshotState state = client + .admin() + .cluster() + .prepareGetSnapshots("test-repo") + .setSnapshots("test-snap") + .get() + .getSnapshots() + .get(0) + .state(); + assertThat(state, equalTo(SnapshotState.SUCCESS)); logger.info("--> delete index"); cluster().wipeIndices("test-idx"); @@ -90,7 +105,14 @@ public void testUrlRepository() throws Exception { .put("url", repositoryLocation.toUri().toURL()) .put("list_directories", randomBoolean()))); logger.info("--> restore index after deletion"); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet(); + RestoreSnapshotResponse restoreSnapshotResponse = client + .admin() + .cluster() + .prepareRestoreSnapshot("url-repo", "test-snap") + .setWaitForCompletion(true) + .setIndices("test-idx") + .execute() + .actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L)); From 63da34db6740f32b4e861d5cfcbccf4dda757357 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 15:34:04 -0600 Subject: [PATCH 18/24] Fix integration test name --- modules/repository-url/build.gradle | 2 +- ...LSnapshotRestoreIT.java => URLSnapshotRestoreTests.java} | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) rename modules/repository-url/src/test/java/org/elasticsearch/repositories/url/{URLSnapshotRestoreIT.java => URLSnapshotRestoreTests.java} (97%) diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index db0165fb25624..67b1ed04cd560 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -19,7 +19,7 @@ esplugin { description 'Module for generic repositories' - classname 'org.elasticsearch.plugin.repositories.RepositoriesPlugin' + classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin' } compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java similarity index 97% rename from modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java rename to modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java index 74a0e76b572f9..fdca4665f7d2b 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java @@ -41,11 +41,7 @@ import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) -public class URLSnapshotRestoreIT extends ESIntegTestCase { - - protected boolean ignoreExternalCluster() { - return true; - } +public class URLSnapshotRestoreTests extends ESIntegTestCase { @Override protected Collection> nodePlugins() { From b49a346d41a3def7d15f80522f2a033fd6c369ea Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 16:27:15 -0600 Subject: [PATCH 19/24] replace rest test using url repo with fs --- .../gradle/test/ClusterFormationTasks.groovy | 3 +-- .../test/snapshot.get_repository/10_basic.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 756c05b07d523..3fc622ef5aa4f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -276,8 +276,7 @@ class ClusterFormationTasks { 'path.repo' : "${node.sharedDir}/repo", 'path.shared_data' : "${node.sharedDir}/", // Define a node attribute so we can test that it exists - 'node.attr.testattr' : 'test', - 'repositories.url.allowed_urls': 'http://snapshot.test*' + 'node.attr.testattr' : 'test' ] // we set min master nodes to the total number of nodes in the cluster and // basically skip initial state recovery to allow the cluster to form using a realistic master election diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml index 5788453a73950..b944fe43791e4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml @@ -5,17 +5,17 @@ setup: snapshot.create_repository: repository: test_repo_get_1 body: - type: url + type: fs settings: - url: "http://snapshot.test1" + location: "test_repo_get_1_loc" - do: snapshot.create_repository: repository: test_repo_get_2 body: - type: url + type: fs settings: - url: "http://snapshot.test2" + location: "test_repo_get_1_loc" --- "Get all repositories": From 1555ca0d7f319f572035a4690a763d1676ce307f Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 17:53:06 -0600 Subject: [PATCH 20/24] Implement basic yaml integration test --- modules/repository-url/build.gradle | 5 +++ .../RepositoryURLClientYamlTestSuiteIT.java | 40 +++++++++++++++++++ .../test/repository_url/10_basic.yaml | 32 +++++++++++++++ .../test/repository_url/20_repository.yaml | 16 ++++++++ 4 files changed, 93 insertions(+) create mode 100644 modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java create mode 100644 modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml create mode 100644 modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index 67b1ed04cd560..ebe058f292458 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -25,3 +25,8 @@ esplugin { compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" +integTest { + cluster { + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' + } +} \ No newline at end of file diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..66fba622e602a --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.url; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +import java.io.IOException; + +public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException { + return ESClientYamlSuiteTestCase.createParameters(); + } +} + diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml new file mode 100644 index 0000000000000..75e7873299869 --- /dev/null +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml @@ -0,0 +1,32 @@ +# Integration tests for URL Repository component +# +"URL Repository plugin loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.modules.0.name: repository-url } + +--- +setup: + + - do: + snapshot.create_repository: + repository: test_repo1 + body: + type: url + settings: + url: "http://snapshot.test1" + + - do: + snapshot.create_repository: + repository: test_repo2 + body: + type: url + settings: + url: "http://snapshot.test2" diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml new file mode 100644 index 0000000000000..39cfeee192c9b --- /dev/null +++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml @@ -0,0 +1,16 @@ +"Repository can be registered": + + - do: + snapshot.create_repository: + repository: test_repo1 + body: + type: url + settings: + url: "http://snapshot.test1" + - is_true: acknowledged + + - do: + snapshot.get_repository: + repository: test_repo1 + + - is_true : test_repo1 From 9ac5852041bfefeb8b494a3ff1a9cc892bc3fcbe Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 22:05:59 -0600 Subject: [PATCH 21/24] Fix integration test by setting config --- qa/rolling-upgrade/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 182e6a9f7d947..b03d12fbdf787 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -29,6 +29,7 @@ task oldClusterTest(type: RestIntegTestTask) { numBwcNodes = 2 numNodes = 2 clusterName = 'rolling-upgrade' + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' } systemProperty 'tests.rest.suite', 'old_cluster' } @@ -40,6 +41,7 @@ task mixedClusterTest(type: RestIntegTestTask) { clusterName = 'rolling-upgrade' unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() } dataDir = "${-> oldClusterTest.nodes[1].dataDir}" + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' } systemProperty 'tests.rest.suite', 'mixed_cluster' finalizedBy 'oldClusterTest#node0.stop' @@ -52,6 +54,7 @@ task upgradedClusterTest(type: RestIntegTestTask) { clusterName = 'rolling-upgrade' unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() } dataDir = "${-> oldClusterTest.nodes[0].dataDir}" + setting 'repositories.url.allowed_urls', 'http://snapshot.test*' } systemProperty 'tests.rest.suite', 'upgraded_cluster' // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion From dd03e3a79ce4bd968bd3588b9f467dd9ddde110b Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 24 Jan 2017 23:44:42 -0600 Subject: [PATCH 22/24] Use constants --- .../repositories/url/URLSnapshotRestoreTests.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java index fdca4665f7d2b..f7fb7af055c62 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.plugin.repository.url.URLRepositoryPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ESIntegTestCase; @@ -54,10 +55,10 @@ public void testUrlRepository() throws Exception { logger.info("--> creating repository"); Path repositoryLocation = randomRepoPath(); assertAcked(client.admin().cluster().preparePutRepository("test-repo") - .setType("fs").setSettings(Settings.builder() - .put("location", repositoryLocation) - .put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + .setType(FsRepository.TYPE).setSettings(Settings.builder() + .put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation) + .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean()) + .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); createIndex("test-idx"); ensureGreen(); @@ -98,7 +99,7 @@ public void testUrlRepository() throws Exception { logger.info("--> create read-only URL repository"); assertAcked(client.admin().cluster().preparePutRepository("url-repo") .setType(URLRepository.TYPE).setSettings(Settings.builder() - .put("url", repositoryLocation.toUri().toURL()) + .put(URLRepository.URL_SETTING.getKey(), repositoryLocation.toUri().toURL()) .put("list_directories", randomBoolean()))); logger.info("--> restore index after deletion"); RestoreSnapshotResponse restoreSnapshotResponse = client From 12775b4c3c8b009263676fb00d58bf51c9c1958d Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 25 Jan 2017 13:45:55 -0600 Subject: [PATCH 23/24] Fix module description --- modules/repository-url/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index ebe058f292458..d607b6b540af2 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -18,7 +18,7 @@ */ esplugin { - description 'Module for generic repositories' + description 'Module for URL repository' classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin' } From 0880f6c71f56a6fe8e46a5dbd97ced190dc6ff1f Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 25 Jan 2017 15:57:42 -0600 Subject: [PATCH 24/24] Remove compiler lint args --- modules/repository-url/build.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index d607b6b540af2..dc769e3c1f300 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -22,9 +22,6 @@ esplugin { classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin' } -compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" -compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" - integTest { cluster { setting 'repositories.url.allowed_urls', 'http://snapshot.test*'