From 4055c3476927b537a577fe9443dd7a3956c3da31 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 8 May 2018 20:02:07 +0200 Subject: [PATCH 1/7] Automatic snapshot naming (#7939) --- docs/reference/modules/snapshots.asciidoc | 3 ++- .../create/TransportCreateSnapshotAction.java | 7 ++--- .../cluster/snapshots/SnapshotBlocksIT.java | 27 +++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 693d537d732c1..7c332494dd7e7 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -282,7 +282,8 @@ PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true // TEST[continued] The list of indices that should be included into the snapshot can be specified using the `indices` parameter that -supports <>. The snapshot request also supports the +supports <>. It can be useful to use <> to name the +snapshot according to the date that the snapshot made, e.g. `snapshot-2018.05.09`. The snapshot request also supports the `ignore_unavailable` option. Setting it to `true` will cause indices that do not exist to be ignored during snapshot creation. By default, when `ignore_unavailable` option is not set and an index is missing the snapshot request will fail. By setting `include_global_state` to false it's possible to prevent the cluster global state to be stored as part of diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java index 52fe03f58c28d..a7a5548552be2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java @@ -71,8 +71,9 @@ protected ClusterBlockException checkBlock(CreateSnapshotRequest request, Cluste @Override protected void masterOperation(final CreateSnapshotRequest request, ClusterState state, final ActionListener listener) { + final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.snapshot()); SnapshotsService.SnapshotRequest snapshotRequest = - new SnapshotsService.SnapshotRequest(request.repository(), request.snapshot(), "create_snapshot [" + request.snapshot() + "]") + new SnapshotsService.SnapshotRequest(request.repository(), snapshotName, "create_snapshot [" + snapshotName + "]") .indices(request.indices()) .indicesOptions(request.indicesOptions()) .partial(request.partial()) @@ -87,7 +88,7 @@ public void onResponse() { @Override public void onSnapshotCompletion(Snapshot snapshot, SnapshotInfo snapshotInfo) { if (snapshot.getRepository().equals(request.repository()) && - snapshot.getSnapshotId().getName().equals(request.snapshot())) { + snapshot.getSnapshotId().getName().equals(snapshotName)) { listener.onResponse(new CreateSnapshotResponse(snapshotInfo)); snapshotsService.removeListener(this); } @@ -96,7 +97,7 @@ public void onSnapshotCompletion(Snapshot snapshot, SnapshotInfo snapshotInfo) { @Override public void onSnapshotFailure(Snapshot snapshot, Exception e) { if (snapshot.getRepository().equals(request.repository()) && - snapshot.getSnapshotId().getName().equals(request.snapshot())) { + snapshot.getSnapshotId().getName().equals(snapshotName)) { listener.onFailure(e); snapshotsService.removeListener(this); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index c66fa4b244f18..8c56228a4bea3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.RestStatus; @@ -176,4 +177,30 @@ public void testSnapshotStatusWithBlocks() { setClusterReadOnly(false); } } + + public void testSnapshotWithDateMath() { + // This test checks that the Snapshot Status operation is never blocked, even if the cluster is read only. + try { + setClusterReadOnly(true); + + final IndexNameExpressionResolver nameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); + final String snapshotName = ""; + final String expression = nameExpressionResolver.resolveDateMathExpression(snapshotName); + + CreateSnapshotResponse snapshotResponse = + client().admin().cluster().prepareCreateSnapshot(REPOSITORY_NAME, snapshotName) + .setIncludeGlobalState(true) + .setWaitForCompletion(true) + .execute().actionGet(); + assertThat(snapshotResponse.status(), equalTo(RestStatus.OK)); + + SnapshotsStatusResponse response = client().admin().cluster().prepareSnapshotStatus(REPOSITORY_NAME) + .setSnapshots(expression) + .execute().actionGet(); + assertThat(response.getSnapshots(), hasSize(1)); + assertThat(response.getSnapshots().get(0).getState().completed(), equalTo(true)); + } finally { + setClusterReadOnly(false); + } + } } From 4b20260283081a3917063619b1662135b4934f7a Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 11 May 2018 09:38:17 +0200 Subject: [PATCH 2/7] Automatic snapshot naming #7939 move test case to DedicatedClusterSnapshotRestoreIT --- .../cluster/snapshots/SnapshotBlocksIT.java | 25 --------------- .../DedicatedClusterSnapshotRestoreIT.java | 32 +++++++++++++++++++ 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index 8c56228a4bea3..947c4994b1a19 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.RestStatus; @@ -178,29 +177,5 @@ public void testSnapshotStatusWithBlocks() { } } - public void testSnapshotWithDateMath() { - // This test checks that the Snapshot Status operation is never blocked, even if the cluster is read only. - try { - setClusterReadOnly(true); - - final IndexNameExpressionResolver nameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); - final String snapshotName = ""; - final String expression = nameExpressionResolver.resolveDateMathExpression(snapshotName); - CreateSnapshotResponse snapshotResponse = - client().admin().cluster().prepareCreateSnapshot(REPOSITORY_NAME, snapshotName) - .setIncludeGlobalState(true) - .setWaitForCompletion(true) - .execute().actionGet(); - assertThat(snapshotResponse.status(), equalTo(RestStatus.OK)); - - SnapshotsStatusResponse response = client().admin().cluster().prepareSnapshotStatus(REPOSITORY_NAME) - .setSnapshots(expression) - .execute().actionGet(); - assertThat(response.getSnapshots(), hasSize(1)); - assertThat(response.getSnapshots().get(0).getState().completed(), equalTo(true)); - } finally { - setClusterReadOnly(false); - } - } } diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 5341b268544e7..9b3494e1242e9 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; @@ -41,6 +42,7 @@ import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; @@ -68,6 +70,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.admin.cluster.RestClusterStateAction; import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.elasticsearch.snapshots.mockstore.MockRepository; @@ -96,6 +99,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -981,6 +985,34 @@ public void testRestoreShrinkIndex() throws Exception { ensureYellow(); } + public void testSnapshotWithDateMath() { + final String repo = "repo"; + final AdminClient admin = client().admin(); + + final IndexNameExpressionResolver nameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); + final String snapshotName = ""; + final String expression = nameExpressionResolver.resolveDateMathExpression(snapshotName); + + logger.info("--> creating repository"); + assertAcked(admin.cluster().preparePutRepository(repo).setType("fs") + .setSettings(Settings.builder().put("location", randomRepoPath()) + .put("compress", randomBoolean()))); + + logger.info("--> creating date math snapshot"); + CreateSnapshotResponse snapshotResponse = + admin.cluster().prepareCreateSnapshot(repo, snapshotName) + .setIncludeGlobalState(true) + .setWaitForCompletion(true) + .execute().actionGet(); + assertThat(snapshotResponse.status(), equalTo(RestStatus.OK)); + + SnapshotsStatusResponse response = admin.cluster().prepareSnapshotStatus(repo) + .setSnapshots(expression) + .execute().actionGet(); + assertThat(response.getSnapshots(), hasSize(1)); + assertThat(response.getSnapshots().get(0).getState().completed(), equalTo(true)); + } + public static class SnapshottableMetadata extends TestCustomMetaData { public static final String TYPE = "test_snapshottable"; From b168ec89ee6aacae795bd46c494510d3eb554e47 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 11 May 2018 09:53:48 +0200 Subject: [PATCH 3/7] Automatic snapshot naming #7939 extended doc: example is added. referring to indices date math more clearly --- docs/reference/modules/snapshots.asciidoc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 7c332494dd7e7..48b9306f4fbba 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -282,14 +282,26 @@ PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true // TEST[continued] The list of indices that should be included into the snapshot can be specified using the `indices` parameter that -supports <>. It can be useful to use <> to name the -snapshot according to the date that the snapshot made, e.g. `snapshot-2018.05.09`. The snapshot request also supports the +supports <>. The snapshot request also supports the `ignore_unavailable` option. Setting it to `true` will cause indices that do not exist to be ignored during snapshot creation. By default, when `ignore_unavailable` option is not set and an index is missing the snapshot request will fail. By setting `include_global_state` to false it's possible to prevent the cluster global state to be stored as part of the snapshot. By default, the entire snapshot will fail if one or more indices participating in the snapshot don't have all primary shards available. This behaviour can be changed by setting `partial` to `true`. +Snapshot could be named according to the date that the snapshot made in a similar way as indices +using <>, keep in mind all special characters should be URI encoded. + +For example, following command creates a snapshot with a name like `snapshot-2018.05.11`: +[source,js] +----------------------------------- +# PUT /_snapshot/my_backup/ +PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E +----------------------------------- +// CONSOLE +// TEST[continued] + + The index snapshot process is incremental. In the process of making the index snapshot Elasticsearch analyses the list of the index files that are already stored in the repository and copies only files that were created or changed since the last snapshot. That allows multiple snapshots to be preserved in the repository in a compact form. From 256c3c77d372ad0359dac52b349d5b3aa94b3972 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 15 May 2018 12:39:14 +0200 Subject: [PATCH 4/7] #7939 dropped unnecessary new lines in SnapshotBlocksIT --- .../action/admin/cluster/snapshots/SnapshotBlocksIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index 947c4994b1a19..c66fa4b244f18 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -176,6 +176,4 @@ public void testSnapshotStatusWithBlocks() { setClusterReadOnly(false); } } - - } From 117495f00806817148542775e592f10c8f396682 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 15 May 2018 12:40:43 +0200 Subject: [PATCH 5/7] #7939 use simple get() instead of execute().actionGet() --- .../snapshots/DedicatedClusterSnapshotRestoreIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 9b3494e1242e9..624b9c47254bf 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -1003,7 +1003,7 @@ public void testSnapshotWithDateMath() { admin.cluster().prepareCreateSnapshot(repo, snapshotName) .setIncludeGlobalState(true) .setWaitForCompletion(true) - .execute().actionGet(); + .get(); assertThat(snapshotResponse.status(), equalTo(RestStatus.OK)); SnapshotsStatusResponse response = admin.cluster().prepareSnapshotStatus(repo) From ff244bdb02ded0eebf2df9caf0ba4c805afc96e2 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 15 May 2018 12:54:28 +0200 Subject: [PATCH 6/7] #7939 fix test taking into account potential day rollover --- .../DedicatedClusterSnapshotRestoreIT.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 624b9c47254bf..4349f6940cc6a 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -51,6 +51,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; @@ -58,6 +59,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.discovery.zen.ElectMasterService; @@ -991,13 +993,13 @@ public void testSnapshotWithDateMath() { final IndexNameExpressionResolver nameExpressionResolver = new IndexNameExpressionResolver(Settings.EMPTY); final String snapshotName = ""; - final String expression = nameExpressionResolver.resolveDateMathExpression(snapshotName); logger.info("--> creating repository"); assertAcked(admin.cluster().preparePutRepository(repo).setType("fs") .setSettings(Settings.builder().put("location", randomRepoPath()) .put("compress", randomBoolean()))); + final String expression1 = nameExpressionResolver.resolveDateMathExpression(snapshotName); logger.info("--> creating date math snapshot"); CreateSnapshotResponse snapshotResponse = admin.cluster().prepareCreateSnapshot(repo, snapshotName) @@ -1005,12 +1007,16 @@ public void testSnapshotWithDateMath() { .setWaitForCompletion(true) .get(); assertThat(snapshotResponse.status(), equalTo(RestStatus.OK)); + // snapshot could be taken before or after a day rollover + final String expression2 = nameExpressionResolver.resolveDateMathExpression(snapshotName); SnapshotsStatusResponse response = admin.cluster().prepareSnapshotStatus(repo) - .setSnapshots(expression) - .execute().actionGet(); - assertThat(response.getSnapshots(), hasSize(1)); - assertThat(response.getSnapshots().get(0).getState().completed(), equalTo(true)); + .setSnapshots(Sets.newHashSet(expression1, expression2).toArray(Strings.EMPTY_ARRAY)) + .setIgnoreUnavailable(true) + .get(); + List snapshots = response.getSnapshots(); + assertThat(snapshots, hasSize(1)); + assertThat(snapshots.get(0).getState().completed(), equalTo(true)); } public static class SnapshottableMetadata extends TestCustomMetaData { From 46481451657874941e494e62f6d758e0253b5a9e Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 15 May 2018 12:57:14 +0200 Subject: [PATCH 7/7] #7939 doc adjusted --- docs/reference/modules/snapshots.asciidoc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 48b9306f4fbba..f70857e66c86f 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -289,10 +289,11 @@ By setting `include_global_state` to false it's possible to prevent the cluster the snapshot. By default, the entire snapshot will fail if one or more indices participating in the snapshot don't have all primary shards available. This behaviour can be changed by setting `partial` to `true`. -Snapshot could be named according to the date that the snapshot made in a similar way as indices -using <>, keep in mind all special characters should be URI encoded. +Snapshot names can be automatically derived using <>, similarly as when creating +new indices. Note that special characters need to be URI encoded. -For example, following command creates a snapshot with a name like `snapshot-2018.05.11`: +For example, creating a snapshot with the current day in the name, like `snapshot-2018.05.11`, can be achieved with +the following command: [source,js] ----------------------------------- # PUT /_snapshot/my_backup/