From 71a30abf353a7ccf20b4bbf3ff7395315cda1d1b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 16 Jun 2017 16:37:21 -0400 Subject: [PATCH 1/6] WIP --- .../upgrades/FullClusterRestartIT.java | 75 +++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 23ea65b193ce2..f840fe6ff53e8 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.upgrades; +import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; @@ -47,7 +48,6 @@ import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; /** * Tests to run before and after a full cluster restart. This is run twice, @@ -448,9 +448,50 @@ public void testRecovery() throws IOException { public void testSnapshotRestore() throws IOException { int count; if (runningAgainstOldCluster) { + // Create the index count = between(200, 300); indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject()); + // Stick a routing attribute into to cluster settings so we can see it after the restore + HttpEntity routingSetting = new StringEntity( + "{\"persistent\":{\"cluster.routing.allocation.exclude.test_attr\": \"" + oldClusterVersion + "\"}}", + ContentType.APPLICATION_JSON); + client().performRequest("PUT", "/_cluster/settings", emptyMap(), routingSetting); + + // Stick a template into the cluster so we can see it after the restore + XContentBuilder templateBuilder = JsonXContent.contentBuilder().startObject(); + templateBuilder.field("template", "te*"); + templateBuilder.startObject("settings"); { + templateBuilder.field("number_of_shards", 1); + } + templateBuilder.endObject(); + templateBuilder.startObject("mappings"); { + templateBuilder.startObject("doc"); { + templateBuilder.startObject("_source"); { + templateBuilder.field("enabled", randomLenientBoolean()); + } + templateBuilder.endObject(); + } + templateBuilder.endObject(); + } + templateBuilder.endObject(); + templateBuilder.startObject("aliases"); { + templateBuilder.startObject("alias1").endObject(); + templateBuilder.startObject("alias2"); { + templateBuilder.startObject("filter"); { + templateBuilder.startObject("term"); { + templateBuilder.field("version", oldClusterVersion); + } + templateBuilder.endObject(); + } + templateBuilder.endObject(); + } + templateBuilder.endObject(); + } + templateBuilder.endObject().endObject(); + client().performRequest("PUT", "/_template/test_template", emptyMap(), + new StringEntity(templateBuilder.string(), ContentType.APPLICATION_JSON)); + // Create the repo and the snapshot XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject(); { repoConfig.field("type", "fs"); @@ -481,6 +522,18 @@ public void testSnapshotRestore() throws IOException { String countResponse = toStr(client().performRequest("GET", "/" + index + "/_search", singletonMap("size", "0"))); assertThat(countResponse, containsString("\"total\":" + count)); + /* Remove the routing setting and template that we added before the + * snapshot so we can test restoring them. If we're running on the + * new cluster then they should exist because they were leftover from + * the restore done by the old cluster. We leave them in place so we + * get a test of a full cluster restart with templates restored from + * a snapshot. */ + HttpEntity clearRoutingSetting = new StringEntity( + "{\"persistent\":\"cluster.routing.allocation.exclude.test_attr\": null}}", + ContentType.APPLICATION_JSON); + client().performRequest("PUT", "/_cluster/settings", emptyMap(), clearRoutingSetting); + client().performRequest("DELETE", "/_template/test_template", emptyMap(), clearRoutingSetting); + if (false == runningAgainstOldCluster) { /* Remove any "restored" indices from the old cluster run of this test. * We intentionally don't remove them while running this against the @@ -496,8 +549,9 @@ public void testSnapshotRestore() throws IOException { assertEquals(response, singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", map)); assertEquals(response, singletonList(oldClusterVersion.toString()), XContentMapValues.extractValue("snapshots.version", map)); + // Perform the restore XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject(); - restoreCommand.field("include_global_state", randomBoolean()); + restoreCommand.field("include_global_state", true); restoreCommand.field("indices", index); restoreCommand.field("rename_pattern", index); restoreCommand.field("rename_replacement", "restored_" + index); @@ -505,11 +559,22 @@ public void testSnapshotRestore() throws IOException { client().performRequest("POST", "/_snapshot/repo/snap/_restore", singletonMap("wait_for_completion", "true"), new StringEntity(restoreCommand.string(), ContentType.APPLICATION_JSON)); - countResponse = toStr( - client().performRequest("GET", "/restored_" + index + "/_search", singletonMap("size", "0"))); - assertThat(countResponse, containsString("\"total\":" + count)); + // Make sure search finds all documents + countResponse = toStr(client().performRequest("GET", "/restored_" + index + "/_search", singletonMap("size", "0"))); + assertThat(countResponse, containsString("\"total\":" + count)); + + // TODO add documents and count again + + if (false == runningAgainstOldCluster) { + // Check settings added by the restore process + map = toMap(client().performRequest("GET", "/_cluster/settings")); + fail(map.toString()); + + // TODO assert that the template came back too } + } + // TODO tests for upgrades after shrink. We've had trouble with shrink in the past. private void indexRandomDocuments(int count, boolean flushAllowed, boolean saveInfo, From 1a5859cdf762ab50a4a6af340d4c563adfc76caa Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 19 Jun 2017 08:42:06 -0400 Subject: [PATCH 2/6] WIP --- .../upgrades/FullClusterRestartIT.java | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index f840fe6ff53e8..35d44b8045224 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.Before; @@ -83,6 +84,11 @@ protected boolean preserveReposUponCompletion() { return true; } + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + public void testSearch() throws Exception { int count; if (runningAgainstOldCluster) { @@ -306,7 +312,7 @@ void assertRealtimeGetWorks(int count) throws IOException { requestBody = "{ \"query\": { \"match_all\" : {} }}"; Map searchRsp = toMap(client().performRequest("GET", "/" + index + "/_search", Collections.emptyMap(), new StringEntity(requestBody, ContentType.APPLICATION_JSON))); - Map hit = (Map) ((List)(XContentMapValues.extractValue("hits.hits", searchRsp))).get(0); + Map hit = (Map) ((List)(XContentMapValues.extractValue("hits.hits", searchRsp))).get(0); String docId = (String) hit.get("_id"); requestBody = "{ \"doc\" : { \"foo\": \"bar\"}}"; @@ -454,7 +460,7 @@ public void testSnapshotRestore() throws IOException { // Stick a routing attribute into to cluster settings so we can see it after the restore HttpEntity routingSetting = new StringEntity( - "{\"persistent\":{\"cluster.routing.allocation.exclude.test_attr\": \"" + oldClusterVersion + "\"}}", + "{\"persistent\": {\"cluster.routing.allocation.exclude.test_attr\": \"" + oldClusterVersion + "\"}}", ContentType.APPLICATION_JSON); client().performRequest("PUT", "/_cluster/settings", emptyMap(), routingSetting); @@ -529,7 +535,7 @@ public void testSnapshotRestore() throws IOException { * get a test of a full cluster restart with templates restored from * a snapshot. */ HttpEntity clearRoutingSetting = new StringEntity( - "{\"persistent\":\"cluster.routing.allocation.exclude.test_attr\": null}}", + "{\"persistent\":{\"cluster.routing.allocation.exclude.test_attr\": null}}", ContentType.APPLICATION_JSON); client().performRequest("PUT", "/_cluster/settings", emptyMap(), clearRoutingSetting); client().performRequest("DELETE", "/_template/test_template", emptyMap(), clearRoutingSetting); @@ -567,10 +573,27 @@ public void testSnapshotRestore() throws IOException { if (false == runningAgainstOldCluster) { // Check settings added by the restore process - map = toMap(client().performRequest("GET", "/_cluster/settings")); - fail(map.toString()); + map = toMap(client().performRequest("GET", "/_cluster/settings", singletonMap("flat_settings", "true"))); + Map expected = new HashMap<>(); + expected.put("transient", emptyMap()); + expected.put("persistent", singletonMap("cluster.routing.allocation.exclude.test_attr", oldClusterVersion.toString())); + if (false == expected.equals(map)) { + NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); + builder.compareMaps(map, expected); + fail(builder.toString()); + } - // TODO assert that the template came back too + map = toMap(client().performRequest("GET", "/_template/test_template")); + expected = new HashMap<>(); + expected.put("patterns", "te*"); + expected.put("number_of_shards", 1); + expected.put("mappings", singletonMap("doc", singletonMap("_source", singletonMap("enabled", "true")))); + expected = singletonMap("test_template", expected); + if (false == expected.equals(map)) { + NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); + builder.compareMaps(map, expected); + fail(builder.toString()); + } } } From 11dff9099559a960dd2db22450fdf59853dc5fdc Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 19 Jun 2017 12:16:08 -0400 Subject: [PATCH 3/6] Port --- .../bwcompat/RepositoryUpgradabilityIT.java | 204 ----------------- .../bwcompat/RestoreBackwardsCompatIT.java | 119 +--------- .../upgrades/FullClusterRestartIT.java | 210 ++++++++++-------- 3 files changed, 121 insertions(+), 412 deletions(-) delete mode 100644 core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java deleted file mode 100644 index 92c8b2315cc85..0000000000000 --- a/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.bwcompat; - -import org.elasticsearch.Version; -import org.elasticsearch.common.io.FileTestUtils; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; -import org.elasticsearch.snapshots.SnapshotId; -import org.elasticsearch.snapshots.SnapshotInfo; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.junit.annotations.TestLogging; - -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; - -/** - * Tests that a repository can handle both snapshots of previous version formats and new version formats, - * as blob names and repository blob formats have changed between the snapshot versions. - */ -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) -// this test sometimes fails in recovery when the recovery is reset, increasing the logging level to help debug -@TestLogging("org.elasticsearch.indices.recovery:DEBUG") -public class RepositoryUpgradabilityIT extends AbstractSnapshotIntegTestCase { - - /** - * This tests that a repository can inter-operate with snapshots that both have and don't have a UUID, - * namely when a repository was created in an older version with snapshots created in the old format - * (only snapshot name, no UUID) and then the repository is loaded into newer versions where subsequent - * snapshots have a name and a UUID. - */ - public void testRepositoryWorksWithCrossVersions() throws Exception { - final List repoVersions = listRepoVersions(); - // run the test for each supported version - for (final String version : repoVersions) { - final String repoName = "test-repo-" + version; - logger.info("--> creating repository [{}] for version [{}]", repoName, version); - createRepository(version, repoName); - - logger.info("--> get the snapshots"); - final String originalIndex = "index-" + version; - final Set indices = Sets.newHashSet(originalIndex); - final Set snapshotInfos = Sets.newHashSet(getSnapshots(repoName)); - assertThat(snapshotInfos.size(), equalTo(1)); - SnapshotInfo originalSnapshot = snapshotInfos.iterator().next(); - if (Version.fromString(version).before(Version.V_5_0_0_alpha1)) { - assertThat(originalSnapshot.snapshotId(), equalTo(new SnapshotId("test_1", "test_1"))); - } else { - assertThat(originalSnapshot.snapshotId().getName(), equalTo("test_1")); - assertNotNull(originalSnapshot.snapshotId().getUUID()); // it's a random UUID now - } - assertThat(Sets.newHashSet(originalSnapshot.indices()), equalTo(indices)); - - logger.info("--> restore the original snapshot"); - final Set restoredIndices = Sets.newHashSet( - restoreSnapshot(repoName, originalSnapshot.snapshotId().getName()) - ); - assertThat(restoredIndices, equalTo(indices)); - // make sure it has documents - for (final String searchIdx : restoredIndices) { - assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().getTotalHits(), greaterThan(0L)); - } - deleteIndices(restoredIndices); // delete so we can restore again later - - final String snapshotName2 = "test_2"; - logger.info("--> take a new snapshot of the old index"); - final int addedDocSize = 10; - for (int i = 0; i < addedDocSize; i++) { - index(originalIndex, "doc", Integer.toString(i), "foo", "new-bar-" + i); - } - refresh(); - snapshotInfos.add(createSnapshot(repoName, snapshotName2)); - - logger.info("--> get the snapshots with the newly created snapshot [{}]", snapshotName2); - Set snapshotInfosFromRepo = Sets.newHashSet(getSnapshots(repoName)); - assertThat(snapshotInfosFromRepo, equalTo(snapshotInfos)); - snapshotInfosFromRepo.forEach(snapshotInfo -> { - assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(indices)); - }); - - final String snapshotName3 = "test_3"; - final String indexName2 = "index2"; - logger.info("--> take a new snapshot with a new index"); - createIndex(indexName2); - indices.add(indexName2); - for (int i = 0; i < addedDocSize; i++) { - index(indexName2, "doc", Integer.toString(i), "foo", "new-bar-" + i); - } - refresh(); - snapshotInfos.add(createSnapshot(repoName, snapshotName3)); - - logger.info("--> get the snapshots with the newly created snapshot [{}]", snapshotName3); - snapshotInfosFromRepo = Sets.newHashSet(getSnapshots(repoName)); - assertThat(snapshotInfosFromRepo, equalTo(snapshotInfos)); - snapshotInfosFromRepo.forEach(snapshotInfo -> { - if (snapshotInfo.snapshotId().getName().equals(snapshotName3)) { - // only the last snapshot has all the indices - assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(indices)); - } else { - assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(Sets.newHashSet(originalIndex))); - } - }); - deleteIndices(indices); // clean up indices - - logger.info("--> restore the old snapshot again"); - Set oldRestoredIndices = Sets.newHashSet(restoreSnapshot(repoName, originalSnapshot.snapshotId().getName())); - assertThat(oldRestoredIndices, equalTo(Sets.newHashSet(originalIndex))); - for (final String searchIdx : oldRestoredIndices) { - assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().getTotalHits(), - greaterThanOrEqualTo((long)addedDocSize)); - } - deleteIndices(oldRestoredIndices); - - logger.info("--> restore the new snapshot"); - Set newSnapshotIndices = Sets.newHashSet(restoreSnapshot(repoName, snapshotName3)); - assertThat(newSnapshotIndices, equalTo(Sets.newHashSet(originalIndex, indexName2))); - for (final String searchIdx : newSnapshotIndices) { - assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().getTotalHits(), - greaterThanOrEqualTo((long)addedDocSize)); - } - deleteIndices(newSnapshotIndices); // clean up indices before starting again - } - } - - private List listRepoVersions() throws Exception { - final String prefix = "repo"; - final List repoVersions = new ArrayList<>(); - final Path repoFiles = getBwcIndicesPath(); - try (DirectoryStream dirStream = Files.newDirectoryStream(repoFiles, prefix + "-*.zip")) { - for (final Path entry : dirStream) { - final String fileName = entry.getFileName().toString(); - String version = fileName.substring(prefix.length() + 1); - version = version.substring(0, version.length() - ".zip".length()); - repoVersions.add(version); - } - } - return Collections.unmodifiableList(repoVersions); - } - - private void createRepository(final String version, final String repoName) throws Exception { - final String prefix = "repo"; - final Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip"); - final Path repoPath = randomRepoPath(); - FileTestUtils.unzip(repoFile, repoPath, "repo/"); - assertAcked(client().admin().cluster().preparePutRepository(repoName) - .setType("fs") - .setSettings(Settings.builder().put("location", repoPath))); - } - - private List getSnapshots(final String repoName) throws Exception { - return client().admin().cluster().prepareGetSnapshots(repoName) - .addSnapshots("_all") - .get() - .getSnapshots(); - } - - private SnapshotInfo createSnapshot(final String repoName, final String snapshotName) throws Exception { - return client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName) - .setWaitForCompletion(true) - .get() - .getSnapshotInfo(); - } - - private List restoreSnapshot(final String repoName, final String snapshotName) throws Exception { - return client().admin().cluster().prepareRestoreSnapshot(repoName, snapshotName) - .setWaitForCompletion(true) - .get() - .getRestoreInfo() - .indices(); - } - - private void deleteIndices(final Set indices) throws Exception { - client().admin().indices().prepareDelete(indices.toArray(new String[indices.size()])).get(); - } - -} diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java index 9ee8fa654b28f..cd1a133643339 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java @@ -18,27 +18,15 @@ */ package org.elasticsearch.bwcompat; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; -import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.common.io.FileTestUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.repositories.fs.FsRepository; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; -import org.elasticsearch.snapshots.RestoreInfo; -import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotRestoreException; import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.VersionUtils; import org.junit.BeforeClass; import java.io.IOException; @@ -46,19 +34,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Locale; -import java.util.SortedSet; -import java.util.TreeSet; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.notNullValue; +/** + * Tests that restoring from a very old snapshot fails appropriately. + */ @ClusterScope(scope = Scope.TEST) public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase { @@ -77,40 +61,6 @@ public static void repoSetup() throws IOException { repoPath = createTempDir("repositories"); } - public void testRestoreOldSnapshots() throws Exception { - String repo = "test_repo"; - String snapshot = "test_1"; - List repoVersions = repoVersions(); - assertThat(repoVersions.size(), greaterThan(0)); - for (String version : repoVersions) { - createRepo("repo", version, repo); - testOldSnapshot(version, repo, snapshot); - } - - SortedSet expectedVersions = new TreeSet<>(); - for (Version v : VersionUtils.allReleasedVersions()) { - // The current version is in the "released" list even though it isn't released for historical reasons - if (v == Version.CURRENT) continue; - if (v.isRelease() == false) continue; // no guarantees for prereleases - if (v.before(Version.CURRENT.minimumIndexCompatibilityVersion())) continue; // we only support versions N and N-1 - if (v.equals(Version.CURRENT)) continue; // the current version is always compatible with itself - expectedVersions.add(v.toString()); - } - - for (String repoVersion : repoVersions) { - if (expectedVersions.remove(repoVersion) == false) { - logger.warn("Old repositories tests contain extra repo: {}", repoVersion); - } - } - if (expectedVersions.isEmpty() == false) { - StringBuilder msg = new StringBuilder("Old repositories tests are missing versions:"); - for (String expected : expectedVersions) { - msg.append("\n" + expected); - } - fail(msg.toString()); - } - } - public void testRestoreUnsupportedSnapshots() throws Exception { String repo = "test_repo"; String snapshot = "test_1"; @@ -122,10 +72,6 @@ public void testRestoreUnsupportedSnapshots() throws Exception { } } - private List repoVersions() throws Exception { - return listRepoVersions("repo"); - } - private List unsupportedRepoVersions() throws Exception { return listRepoVersions("unsupportedrepo"); } @@ -155,65 +101,6 @@ private void createRepo(String prefix, String version, String repo) throws Excep .put(FsRepository.REPOSITORIES_LOCATION_SETTING.getKey(), fsRepoPath.getParent().relativize(fsRepoPath).resolve("repo").toString()))); } - private void testOldSnapshot(String version, String repo, String snapshot) throws IOException { - logger.info("--> get snapshot and check its version"); - GetSnapshotsResponse getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots(repo).setSnapshots(snapshot).get(); - assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1)); - SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0); - assertThat(snapshotInfo.version().toString(), equalTo(version)); - - logger.info("--> get less verbose snapshot info"); - getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots(repo) - .setSnapshots(snapshot).setVerbose(false).get(); - assertEquals(1, getSnapshotsResponse.getSnapshots().size()); - snapshotInfo = getSnapshotsResponse.getSnapshots().get(0); - assertEquals(snapshot, snapshotInfo.snapshotId().getName()); - assertNull(snapshotInfo.version()); // in verbose=false mode, version doesn't exist - - logger.info("--> restoring snapshot"); - RestoreSnapshotResponse response = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot).setRestoreGlobalState(true).setWaitForCompletion(true).get(); - assertThat(response.status(), equalTo(RestStatus.OK)); - RestoreInfo restoreInfo = response.getRestoreInfo(); - assertThat(restoreInfo.successfulShards(), greaterThan(0)); - assertThat(restoreInfo.successfulShards(), equalTo(restoreInfo.totalShards())); - assertThat(restoreInfo.failedShards(), equalTo(0)); - String index = restoreInfo.indices().get(0); - - logger.info("--> check search"); - SearchResponse searchResponse = client().prepareSearch(index).get(); - assertThat(searchResponse.getHits().getTotalHits(), greaterThan(1L)); - - logger.info("--> check settings"); - ClusterState clusterState = client().admin().cluster().prepareState().get().getState(); - assertThat(clusterState.metaData().persistentSettings().get(FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + "version_attr"), equalTo(version)); - - logger.info("--> check templates"); - IndexTemplateMetaData template = clusterState.getMetaData().templates().get("template_" + version.toLowerCase(Locale.ROOT)); - assertThat(template, notNullValue()); - assertThat(template.patterns(), equalTo(Collections.singletonList("te*"))); - assertThat(template.settings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1), equalTo(1)); - assertThat(template.mappings().size(), equalTo(1)); - assertThat(template.mappings().get("type1").string(), - anyOf( - equalTo("{\"type1\":{\"_source\":{\"enabled\":false}}}"), - equalTo("{\"type1\":{\"_source\":{\"enabled\":\"false\"}}}"), - equalTo("{\"type1\":{\"_source\":{\"enabled\":\"0\"}}}"), - equalTo("{\"type1\":{\"_source\":{\"enabled\":0}}}"), - equalTo("{\"type1\":{\"_source\":{\"enabled\":\"off\"}}}"), - equalTo("{\"type1\":{\"_source\":{\"enabled\":\"no\"}}}") - )); - assertThat(template.aliases().size(), equalTo(3)); - assertThat(template.aliases().get("alias1"), notNullValue()); - assertThat(template.aliases().get("alias2").filter().string(), containsString(version)); - assertThat(template.aliases().get("alias2").indexRouting(), equalTo("kimchy")); - assertThat(template.aliases().get("{index}-alias"), notNullValue()); - - logger.info("--> cleanup"); - cluster().wipeIndices(restoreInfo.indices().toArray(new String[restoreInfo.indices().size()])); - cluster().wipeTemplates(); - - } - private void assertUnsupportedIndexFailsToRestore(String repo, String snapshot) throws IOException { logger.info("--> restoring unsupported snapshot"); try { diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 35d44b8045224..eee6d8e948413 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -253,7 +253,7 @@ void assertAllSearchWorks(int count) throws IOException { assertNoFailures(searchRsp); int totalHits = (int) XContentMapValues.extractValue("hits.total", searchRsp); assertEquals(count, totalHits); - Map bestHit = (Map) ((List)(XContentMapValues.extractValue("hits.hits", searchRsp))).get(0); + Map bestHit = (Map) ((List)(XContentMapValues.extractValue("hits.hits", searchRsp))).get(0); // Make sure there are payloads and they are taken into account for the score // the 'string' field has a boost of 4 in the mappings so it should get a payload boost @@ -451,54 +451,74 @@ public void testRecovery() throws IOException { } } + /** + * Tests snapshot/restore by creating a snapshot and restoring it. It takes + * a snapshot on the old cluster and restores it on the old cluster as a + * sanity check and on the new cluster as an upgrade test. It also takes a + * snapshot on the new cluster and restores that on the new cluster as a + * test that the repository is ok with containing snapshot from both the + * old and new versions. All of the snapshots include an index, a template, + * and some routing configuration. + */ public void testSnapshotRestore() throws IOException { int count; if (runningAgainstOldCluster) { // Create the index count = between(200, 300); indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject()); + } else { + count = countOfIndexedRandomDocuments(); + } - // Stick a routing attribute into to cluster settings so we can see it after the restore - HttpEntity routingSetting = new StringEntity( - "{\"persistent\": {\"cluster.routing.allocation.exclude.test_attr\": \"" + oldClusterVersion + "\"}}", - ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/_cluster/settings", emptyMap(), routingSetting); - - // Stick a template into the cluster so we can see it after the restore - XContentBuilder templateBuilder = JsonXContent.contentBuilder().startObject(); - templateBuilder.field("template", "te*"); - templateBuilder.startObject("settings"); { - templateBuilder.field("number_of_shards", 1); - } - templateBuilder.endObject(); - templateBuilder.startObject("mappings"); { - templateBuilder.startObject("doc"); { - templateBuilder.startObject("_source"); { - templateBuilder.field("enabled", randomLenientBoolean()); - } - templateBuilder.endObject(); + // Refresh the index so the count doesn't fail + refresh(); + + // Count the documents in the index to make sure we have as many as we put there + String countResponse = toStr(client().performRequest("GET", "/" + index + "/_search", singletonMap("size", "0"))); + assertThat(countResponse, containsString("\"total\":" + count)); + + // Stick a routing attribute into to cluster settings so we can see it after the restore + HttpEntity routingSetting = new StringEntity( + "{\"persistent\": {\"cluster.routing.allocation.exclude.test_attr\": \"" + oldClusterVersion + "\"}}", + ContentType.APPLICATION_JSON); + client().performRequest("PUT", "/_cluster/settings", emptyMap(), routingSetting); + + // Stick a template into the cluster so we can see it after the restore + XContentBuilder templateBuilder = JsonXContent.contentBuilder().startObject(); + templateBuilder.field("template", "evil_*"); // Don't confuse other tests by applying the template + templateBuilder.startObject("settings"); { + templateBuilder.field("number_of_shards", 1); + } + templateBuilder.endObject(); + templateBuilder.startObject("mappings"); { + templateBuilder.startObject("doc"); { + templateBuilder.startObject("_source"); { + templateBuilder.field("enabled", true); } templateBuilder.endObject(); } templateBuilder.endObject(); - templateBuilder.startObject("aliases"); { - templateBuilder.startObject("alias1").endObject(); - templateBuilder.startObject("alias2"); { - templateBuilder.startObject("filter"); { - templateBuilder.startObject("term"); { - templateBuilder.field("version", oldClusterVersion); - } - templateBuilder.endObject(); + } + templateBuilder.endObject(); + templateBuilder.startObject("aliases"); { + templateBuilder.startObject("alias1").endObject(); + templateBuilder.startObject("alias2"); { + templateBuilder.startObject("filter"); { + templateBuilder.startObject("term"); { + templateBuilder.field("version", runningAgainstOldCluster ? oldClusterVersion : Version.CURRENT); } templateBuilder.endObject(); } templateBuilder.endObject(); } - templateBuilder.endObject().endObject(); - client().performRequest("PUT", "/_template/test_template", emptyMap(), - new StringEntity(templateBuilder.string(), ContentType.APPLICATION_JSON)); + templateBuilder.endObject(); + } + templateBuilder.endObject().endObject(); + client().performRequest("PUT", "/_template/test_template", emptyMap(), + new StringEntity(templateBuilder.string(), ContentType.APPLICATION_JSON)); - // Create the repo and the snapshot + if (runningAgainstOldCluster) { + // Create the repo XContentBuilder repoConfig = JsonXContent.contentBuilder().startObject(); { repoConfig.field("type", "fs"); repoConfig.startObject("settings"); { @@ -510,90 +530,96 @@ public void testSnapshotRestore() throws IOException { repoConfig.endObject(); client().performRequest("PUT", "/_snapshot/repo", emptyMap(), new StringEntity(repoConfig.string(), ContentType.APPLICATION_JSON)); + } - XContentBuilder snapshotConfig = JsonXContent.contentBuilder().startObject(); { - snapshotConfig.field("indices", index); - } - snapshotConfig.endObject(); - client().performRequest("PUT", "/_snapshot/repo/snap", singletonMap("wait_for_completion", "true"), - new StringEntity(snapshotConfig.string(), ContentType.APPLICATION_JSON)); + client().performRequest("PUT", "/_snapshot/repo/" + (runningAgainstOldCluster ? "old_snap" : "new_snap"), + singletonMap("wait_for_completion", "true"), + new StringEntity("{\"indices\": \"" + index + "\"}", ContentType.APPLICATION_JSON)); - // Refresh the index so the count doesn't fail - refresh(); - } else { - count = countOfIndexedRandomDocuments(); + checkSnapshot("old_snap", count, oldClusterVersion); + if (false == runningAgainstOldCluster) { + checkSnapshot("new_snap", count, Version.CURRENT); } + } - // Count the documents in the index to make sure we have as many as we put there - String countResponse = toStr(client().performRequest("GET", "/" + index + "/_search", singletonMap("size", "0"))); - assertThat(countResponse, containsString("\"total\":" + count)); + private void checkSnapshot(String snapshotName, int count, Version tookOnVersion) throws IOException { + // Check the snapshot metadata, especially the version + String response = toStr(client().performRequest("GET", "/_snapshot/repo/" + snapshotName, singletonMap("verbose", "true"))); + Map map = toMap(response); + assertEquals(response, singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", map)); + assertEquals(response, singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", map)); + assertEquals(response, singletonList(tookOnVersion.toString()), XContentMapValues.extractValue("snapshots.version", map)); - /* Remove the routing setting and template that we added before the - * snapshot so we can test restoring them. If we're running on the - * new cluster then they should exist because they were leftover from - * the restore done by the old cluster. We leave them in place so we - * get a test of a full cluster restart with templates restored from - * a snapshot. */ + // Remove the routing setting and template so we can test restoring them. HttpEntity clearRoutingSetting = new StringEntity( "{\"persistent\":{\"cluster.routing.allocation.exclude.test_attr\": null}}", ContentType.APPLICATION_JSON); client().performRequest("PUT", "/_cluster/settings", emptyMap(), clearRoutingSetting); client().performRequest("DELETE", "/_template/test_template", emptyMap(), clearRoutingSetting); - if (false == runningAgainstOldCluster) { - /* Remove any "restored" indices from the old cluster run of this test. - * We intentionally don't remove them while running this against the - * old cluster so we can test starting the node with a restored index - * in the cluster. */ - client().performRequest("DELETE", "/restored_*"); - } - - // Check the metadata, especially the version - String response = toStr(client().performRequest("GET", "/_snapshot/repo/_all", singletonMap("verbose", "true"))); - Map map = toMap(response); - assertEquals(response, singletonList("snap"), XContentMapValues.extractValue("snapshots.snapshot", map)); - assertEquals(response, singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", map)); - assertEquals(response, singletonList(oldClusterVersion.toString()), XContentMapValues.extractValue("snapshots.version", map)); - - // Perform the restore + // Restore XContentBuilder restoreCommand = JsonXContent.contentBuilder().startObject(); restoreCommand.field("include_global_state", true); restoreCommand.field("indices", index); restoreCommand.field("rename_pattern", index); restoreCommand.field("rename_replacement", "restored_" + index); restoreCommand.endObject(); - client().performRequest("POST", "/_snapshot/repo/snap/_restore", singletonMap("wait_for_completion", "true"), + client().performRequest("POST", "/_snapshot/repo/" + snapshotName + "/_restore", singletonMap("wait_for_completion", "true"), new StringEntity(restoreCommand.string(), ContentType.APPLICATION_JSON)); // Make sure search finds all documents - countResponse = toStr(client().performRequest("GET", "/restored_" + index + "/_search", singletonMap("size", "0"))); + String countResponse = toStr(client().performRequest("GET", "/restored_" + index + "/_search", singletonMap("size", "0"))); assertThat(countResponse, containsString("\"total\":" + count)); - // TODO add documents and count again + // Add some extra documents to the index to be sure we can still write to it after restoring it + int extras = between(1, 100); + StringBuilder bulk = new StringBuilder(); + for (int i = 0; i < extras; i++) { + bulk.append("{\"index\":{\"_id\":\"").append(count + i).append("\"}}\n"); + bulk.append("{\"test\":\"test\"}\n"); + } + client().performRequest("POST", "/restored_" + index + "/doc/_bulk", singletonMap("refresh", "true"), + new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); - if (false == runningAgainstOldCluster) { - // Check settings added by the restore process - map = toMap(client().performRequest("GET", "/_cluster/settings", singletonMap("flat_settings", "true"))); - Map expected = new HashMap<>(); - expected.put("transient", emptyMap()); - expected.put("persistent", singletonMap("cluster.routing.allocation.exclude.test_attr", oldClusterVersion.toString())); - if (false == expected.equals(map)) { - NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); - builder.compareMaps(map, expected); - fail(builder.toString()); - } + // And count to make sure the add worked + // Make sure search finds all documents + countResponse = toStr(client().performRequest("GET", "/restored_" + index + "/_search", singletonMap("size", "0"))); + assertThat(countResponse, containsString("\"total\":" + (count + extras))); + + // Clean up the index for the next iteration + client().performRequest("DELETE", "/restored_*"); + + // Check settings added by the restore process + map = toMap(client().performRequest("GET", "/_cluster/settings", singletonMap("flat_settings", "true"))); + Map expected = new HashMap<>(); + expected.put("transient", emptyMap()); + expected.put("persistent", singletonMap("cluster.routing.allocation.exclude.test_attr", oldClusterVersion.toString())); + if (false == expected.equals(map)) { + NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); + builder.compareMaps(map, expected); + fail("settings don't match:\n" + builder.toString()); + } - map = toMap(client().performRequest("GET", "/_template/test_template")); - expected = new HashMap<>(); - expected.put("patterns", "te*"); - expected.put("number_of_shards", 1); - expected.put("mappings", singletonMap("doc", singletonMap("_source", singletonMap("enabled", "true")))); - expected = singletonMap("test_template", expected); - if (false == expected.equals(map)) { - NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); - builder.compareMaps(map, expected); - fail(builder.toString()); - } + // Check that the template was restored successfully + map = toMap(client().performRequest("GET", "/_template/test_template")); + expected = new HashMap<>(); + if (runningAgainstOldCluster) { + expected.put("template", "evil_*"); + } else { + expected.put("index_patterns", singletonList("evil_*")); + } + expected.put("settings", singletonMap("index", singletonMap("number_of_shards", "1"))); + expected.put("mappings", singletonMap("doc", singletonMap("_source", singletonMap("enabled", true)))); + expected.put("order", 0); + Map aliases = new HashMap<>(); + aliases.put("alias1", emptyMap()); + aliases.put("alias2", singletonMap("filter", singletonMap("term", singletonMap("version", tookOnVersion.toString())))); + expected.put("aliases", aliases); + expected = singletonMap("test_template", expected); + if (false == expected.equals(map)) { + NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); + builder.compareMaps(map, expected); + fail("template doesn't match:\n" + builder.toString()); } } From d3415b438d0e65c1354d6aeb83b96ed85fa82246 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 19 Jun 2017 12:25:54 -0400 Subject: [PATCH 4/6] Enable --- qa/full-cluster-restart/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index 6554962d4f70b..8759cac4157bd 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -79,7 +79,7 @@ for (Version version : indexCompatVersions) { dependsOn = [upgradedClusterTest] } - if (project.bwc_tests_enabled == false) { + if (project.bwc_tests_enabled) { bwcTest.dependsOn(versionBwcTest) } } From fffaf0254f0e1c195db13b586a2a06c2381f3d84 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 19 Jun 2017 12:31:28 -0400 Subject: [PATCH 5/6] Fix for < 5.5 --- .../upgrades/FullClusterRestartIT.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index eee6d8e948413..24e196d5868ee 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -544,7 +544,7 @@ public void testSnapshotRestore() throws IOException { private void checkSnapshot(String snapshotName, int count, Version tookOnVersion) throws IOException { // Check the snapshot metadata, especially the version - String response = toStr(client().performRequest("GET", "/_snapshot/repo/" + snapshotName, singletonMap("verbose", "true"))); + String response = toStr(client().performRequest("GET", "/_snapshot/repo/" + snapshotName, listSnapshotVerboseParams())); Map map = toMap(response); assertEquals(response, singletonList(snapshotName), XContentMapValues.extractValue("snapshots.snapshot", map)); assertEquals(response, singletonList("SUCCESS"), XContentMapValues.extractValue("snapshots.state", map)); @@ -624,6 +624,17 @@ private void checkSnapshot(String snapshotName, int count, Version tookOnVersion } + /** + * Parameters required to get the version of Elasticsearch that took the snapshot. + * On versions after 5.5 we need a {@code verbose} parameter. + */ + private Map listSnapshotVerboseParams() { + if (runningAgainstOldCluster && oldClusterVersion.before(Version.V_5_5_0)) { + return emptyMap(); + } + return singletonMap("verbose", "true"); + } + // TODO tests for upgrades after shrink. We've had trouble with shrink in the past. private void indexRandomDocuments(int count, boolean flushAllowed, boolean saveInfo, From 638ba35858f8688c16371c7494a4a0011f73c16d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 21 Jun 2017 12:25:45 -0400 Subject: [PATCH 6/6] Flip --- .../java/org/elasticsearch/upgrades/FullClusterRestartIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index cae0f948f0fd1..d6e7c88aba832 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -746,7 +746,7 @@ private void checkSnapshot(String snapshotName, int count, Version tookOnVersion Map expected = new HashMap<>(); expected.put("transient", emptyMap()); expected.put("persistent", singletonMap("cluster.routing.allocation.exclude.test_attr", oldClusterVersion.toString())); - if (false == expected.equals(map)) { + if (expected.equals(map) == false) { NotEqualMessageBuilder builder = new NotEqualMessageBuilder(); builder.compareMaps(map, expected); fail("settings don't match:\n" + builder.toString());