diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index 616850c513af7..afc7a88151ebe 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -41,9 +41,12 @@ import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.RestoreInfo; +import org.elasticsearch.snapshots.SnapshotInfo; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.stream.Collectors; import static org.hamcrest.Matchers.contains; @@ -139,6 +142,9 @@ public void testCreateSnapshot() throws IOException { CreateSnapshotRequest request = new CreateSnapshotRequest(repository, snapshot); boolean waitForCompletion = randomBoolean(); request.waitForCompletion(waitForCompletion); + if (randomBoolean()) { + request.userMetadata(randomUserMetadata()); + } request.partial(randomBoolean()); request.includeGlobalState(randomBoolean()); @@ -167,6 +173,8 @@ public void testGetSnapshots() throws IOException { CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1); CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2); createSnapshotRequest2.waitForCompletion(true); + Map originalMetadata = randomUserMetadata(); + createSnapshotRequest2.userMetadata(originalMetadata); CreateSnapshotResponse putSnapshotResponse2 = createTestSnapshot(createSnapshotRequest2); // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead. assertEquals(RestStatus.OK, putSnapshotResponse1.status()); @@ -186,6 +194,12 @@ public void testGetSnapshots() throws IOException { assertEquals(2, response.getSnapshots().size()); assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()), contains("test_snapshot1", "test_snapshot2")); + response.getSnapshots().stream() + .filter(s -> s.snapshotId().getName().equals("test_snapshot2")) + .findFirst() + .map(SnapshotInfo::userMetadata) + .ifPresentOrElse(metadata -> assertEquals(originalMetadata, metadata), + () -> assertNull("retrieved metadata is null, expected non-null metadata", originalMetadata)); } public void testSnapshotsStatus() throws IOException { @@ -231,6 +245,9 @@ public void testRestoreSnapshot() throws IOException { CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(testRepository, testSnapshot); createSnapshotRequest.indices(testIndex); createSnapshotRequest.waitForCompletion(true); + if (randomBoolean()) { + createSnapshotRequest.userMetadata(randomUserMetadata()); + } CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest); assertEquals(RestStatus.OK, createSnapshotResponse.status()); @@ -261,6 +278,9 @@ public void testDeleteSnapshot() throws IOException { CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(repository, snapshot); createSnapshotRequest.waitForCompletion(true); + if (randomBoolean()) { + createSnapshotRequest.userMetadata(randomUserMetadata()); + } CreateSnapshotResponse createSnapshotResponse = createTestSnapshot(createSnapshotRequest); // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead. assertEquals(RestStatus.OK, createSnapshotResponse.status()); @@ -270,4 +290,28 @@ public void testDeleteSnapshot() throws IOException { assertTrue(response.isAcknowledged()); } + + private static Map randomUserMetadata() { + if (randomBoolean()) { + return null; + } + + Map metadata = new HashMap<>(); + long fields = randomLongBetween(0, 4); + for (int i = 0; i < fields; i++) { + if (randomBoolean()) { + metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)), + randomAlphaOfLengthBetween(5, 5)); + } else { + Map nested = new HashMap<>(); + long nestedFields = randomLongBetween(0, 4); + for (int j = 0; j < nestedFields; j++) { + nested.put(randomValueOtherThanMany(nested::containsKey, () -> randomAlphaOfLengthBetween(2,10)), + randomAlphaOfLengthBetween(5, 5)); + } + metadata.put(randomValueOtherThanMany(metadata::containsKey, () -> randomAlphaOfLengthBetween(2,10)), nested); + } + } + return metadata; + } } diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index ec7916d5a3445..c8874c5fa1d32 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -349,7 +349,11 @@ PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true { "indices": "index_1,index_2", "ignore_unavailable": true, - "include_global_state": false + "include_global_state": false, + "_meta": { + "taken_by": "kimchy", + "taken_because": "backup before upgrading" + } } ----------------------------------- // CONSOLE @@ -363,6 +367,9 @@ By setting `include_global_state` to false it's possible to prevent the cluster the snapshot. By default, the entire snapshot will fail if one or more indices participating in the snapshot don't have all primary shards available. This behaviour can be changed by setting `partial` to `true`. +The `_meta` field can be used to attach arbitrary metadata to the snapshot. This may be a record of who took the snapshot, +why it was taken, or any other data that might be useful. + Snapshot names can be automatically derived using <>, similarly as when creating new indices. Note that special characters need to be URI encoded. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml index e2b7279f1cd68..3b6ccc35df8d2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -87,6 +87,7 @@ setup: - is_false: snapshots.0.failures - is_false: snapshots.0.shards - is_false: snapshots.0.version + - is_false: snapshots.0._meta - do: snapshot.delete: @@ -149,3 +150,41 @@ setup: snapshot.delete: repository: test_repo_get_1 snapshot: test_snapshot_without_include_global_state + +--- +"Get snapshot info with metadata": + - skip: + version: " - 7.9.99" + reason: "https://github.com/elastic/elasticsearch/pull/41281 not yet backported to 7.x" + + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + snapshot.create: + repository: test_repo_get_1 + snapshot: test_snapshot_with_metadata + wait_for_completion: true + body: | + { "metadata": {"taken_by": "test", "foo": {"bar": "baz"}} } + + - do: + snapshot.get: + repository: test_repo_get_1 + snapshot: test_snapshot_with_metadata + + - is_true: snapshots + - match: { snapshots.0.snapshot: test_snapshot_with_metadata } + - match: { snapshots.0.state: SUCCESS } + - match: { snapshots.0.metadata.taken_by: test } + - match: { snapshots.0.metadata.foo.bar: baz } + + - do: + snapshot.delete: + repository: test_repo_get_1 + snapshot: test_snapshot_with_metadata diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java index 8f83da053b215..e0250da5ed1cb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java @@ -19,12 +19,14 @@ package org.elasticsearch.action.admin.cluster.snapshots.create; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -46,6 +48,7 @@ import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; +import static org.elasticsearch.snapshots.SnapshotInfo.METADATA_FIELD_INTRODUCED; /** * Create snapshot request @@ -63,6 +66,7 @@ */ public class CreateSnapshotRequest extends MasterNodeRequest implements IndicesRequest.Replaceable, ToXContentObject { + public static int MAXIMUM_METADATA_BYTES = 1024; // chosen arbitrarily private String snapshot; @@ -80,6 +84,8 @@ public class CreateSnapshotRequest extends MasterNodeRequest userMetadata; + public CreateSnapshotRequest() { } @@ -104,6 +110,9 @@ public CreateSnapshotRequest(StreamInput in) throws IOException { includeGlobalState = in.readBoolean(); waitForCompletion = in.readBoolean(); partial = in.readBoolean(); + if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + userMetadata = in.readMap(); + } } @Override @@ -117,6 +126,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(includeGlobalState); out.writeBoolean(waitForCompletion); out.writeBoolean(partial); + if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + out.writeMap(userMetadata); + } } @Override @@ -144,9 +156,28 @@ public ActionRequestValidationException validate() { if (settings == null) { validationException = addValidationError("settings is null", validationException); } + final int metadataSize = metadataSize(userMetadata); + if (metadataSize > MAXIMUM_METADATA_BYTES) { + validationException = addValidationError("metadata must be smaller than 1024 bytes, but was [" + metadataSize + "]", + validationException); + } return validationException; } + private static int metadataSize(Map userMetadata) { + if (userMetadata == null) { + return 0; + } + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.value(userMetadata); + int size = BytesReference.bytes(builder).length(); + return size; + } catch (IOException e) { + // This should not be possible as we are just rendering the xcontent in memory + throw new ElasticsearchException(e); + } + } + /** * Sets the snapshot name * @@ -378,6 +409,15 @@ public boolean includeGlobalState() { return includeGlobalState; } + public Map userMetadata() { + return userMetadata; + } + + public CreateSnapshotRequest userMetadata(Map userMetadata) { + this.userMetadata = userMetadata; + return this; + } + /** * Parses snapshot definition. * @@ -405,6 +445,11 @@ public CreateSnapshotRequest source(Map source) { settings((Map) entry.getValue()); } else if (name.equals("include_global_state")) { includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state"); + } else if (name.equals("metadata")) { + if (entry.getValue() != null && (entry.getValue() instanceof Map == false)) { + throw new IllegalArgumentException("malformed metadata, should be an object"); + } + userMetadata((Map) entry.getValue()); } } indicesOptions(IndicesOptions.fromMap(source, indicesOptions)); @@ -433,6 +478,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (indicesOptions != null) { indicesOptions.toXContent(builder, params); } + builder.field("metadata", userMetadata); builder.endObject(); return builder; } @@ -460,12 +506,14 @@ public boolean equals(Object o) { Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) && Objects.equals(settings, that.settings) && - Objects.equals(masterNodeTimeout, that.masterNodeTimeout); + Objects.equals(masterNodeTimeout, that.masterNodeTimeout) && + Objects.equals(userMetadata, that.userMetadata); } @Override public int hashCode() { - int result = Objects.hash(snapshot, repository, indicesOptions, partial, settings, includeGlobalState, waitForCompletion); + int result = Objects.hash(snapshot, repository, indicesOptions, partial, settings, includeGlobalState, + waitForCompletion, userMetadata); result = 31 * result + Arrays.hashCode(indices); return result; } @@ -482,6 +530,7 @@ public String toString() { ", includeGlobalState=" + includeGlobalState + ", waitForCompletion=" + waitForCompletion + ", masterNodeTimeout=" + masterNodeTimeout + + ", metadata=" + userMetadata + '}'; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 6f757cb60ca86..62ddc5b7d9df3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -117,4 +118,9 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(snapshots); } + + @Override + public String toString() { + return Strings.toString(this); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index ae9506706e36a..039c7ec447767 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -44,6 +44,8 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.snapshots.SnapshotInfo.METADATA_FIELD_INTRODUCED; + /** * Meta data about snapshots that are currently executing */ @@ -84,11 +86,12 @@ public static class Entry { private final ImmutableOpenMap> waitingIndices; private final long startTime; private final long repositoryStateId; + @Nullable private final Map userMetadata; @Nullable private final String failure; public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, long startTime, long repositoryStateId, ImmutableOpenMap shards, - String failure) { + String failure, Map userMetadata) { this.state = state; this.snapshot = snapshot; this.includeGlobalState = includeGlobalState; @@ -104,21 +107,23 @@ public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, Sta } this.repositoryStateId = repositoryStateId; this.failure = failure; + this.userMetadata = userMetadata; } public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List indices, - long startTime, long repositoryStateId, ImmutableOpenMap shards) { - this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null); + long startTime, long repositoryStateId, ImmutableOpenMap shards, + Map userMetadata) { + this(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards, null, userMetadata); } public Entry(Entry entry, State state, ImmutableOpenMap shards) { this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, - entry.repositoryStateId, shards, entry.failure); + entry.repositoryStateId, shards, entry.failure, entry.userMetadata); } public Entry(Entry entry, State state, ImmutableOpenMap shards, String failure) { this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, - entry.repositoryStateId, shards, failure); + entry.repositoryStateId, shards, failure, entry.userMetadata); } public Entry(Entry entry, ImmutableOpenMap shards) { @@ -149,6 +154,10 @@ public boolean includeGlobalState() { return includeGlobalState; } + public Map userMetadata() { + return userMetadata; + } + public boolean partial() { return partial; } @@ -419,6 +428,10 @@ public SnapshotsInProgress(StreamInput in) throws IOException { } long repositoryStateId = in.readLong(); final String failure = in.readOptionalString(); + Map userMetadata = null; + if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + userMetadata = in.readMap(); + } entries[i] = new Entry(snapshot, includeGlobalState, partial, @@ -427,7 +440,9 @@ public SnapshotsInProgress(StreamInput in) throws IOException { startTime, repositoryStateId, builder.build(), - failure); + failure, + userMetadata + ); } this.entries = Arrays.asList(entries); } @@ -452,6 +467,9 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeLong(entry.repositoryStateId); out.writeOptionalString(entry.failure); + if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + out.writeMap(entry.userMetadata); + } } } diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index 1fa42579617e1..8c9eff0698835 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; public class FilterRepository implements Repository { @@ -79,9 +80,10 @@ public void initializeSnapshot(SnapshotId snapshotId, List indices, Met @Override public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List indices, long startTime, String failure, int totalShards, - List shardFailures, long repositoryStateId, boolean includeGlobalState) { + List shardFailures, long repositoryStateId, boolean includeGlobalState, + Map userMetadata) { return in.finalizeSnapshot(snapshotId, indices, startTime, failure, totalShards, shardFailures, repositoryStateId, - includeGlobalState); + includeGlobalState, userMetadata); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index 0eca92039fbf8..788459b16c540 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.function.Function; /** @@ -134,7 +135,8 @@ default Repository create(RepositoryMetaData metaData, Function indices, long startTime, String failure, int totalShards, - List shardFailures, long repositoryStateId, boolean includeGlobalState); + List shardFailures, long repositoryStateId, boolean includeGlobalState, + Map userMetadata); /** * Deletes snapshot diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 1cb50f0f1a0da..18683c2101f44 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -489,11 +489,12 @@ public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, final int totalShards, final List shardFailures, final long repositoryStateId, - final boolean includeGlobalState) { + final boolean includeGlobalState, + final Map userMetadata) { SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId, indices.stream().map(IndexId::getName).collect(Collectors.toList()), startTime, failure, System.currentTimeMillis(), totalShards, shardFailures, - includeGlobalState); + includeGlobalState, userMetadata); try { snapshotFormat.write(blobStoreSnapshot, blobContainer(), snapshotId.getUUID()); final RepositoryData repositoryData = getRepositoryData(); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index 38aa945bcca47..484f116367ba8 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -42,6 +42,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -51,6 +52,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, public static final String CONTEXT_MODE_PARAM = "context_mode"; public static final String CONTEXT_MODE_SNAPSHOT = "SNAPSHOT"; + public static final Version METADATA_FIELD_INTRODUCED = Version.V_8_0_0; // TODO Set this to the earliest version this is backported to private static final DateFormatter DATE_TIME_FORMATTER = DateFormatter.forPattern("strictDateOptionalTime"); private static final String SNAPSHOT = "snapshot"; private static final String UUID = "uuid"; @@ -74,6 +76,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, private static final String TOTAL_SHARDS = "total_shards"; private static final String SUCCESSFUL_SHARDS = "successful_shards"; private static final String INCLUDE_GLOBAL_STATE = "include_global_state"; + private static final String USER_METADATA = "metadata"; private static final Comparator COMPARATOR = Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId); @@ -88,6 +91,7 @@ public static final class SnapshotInfoBuilder { private long endTime = 0L; private ShardStatsBuilder shardStatsBuilder = null; private Boolean includeGlobalState = null; + private Map userMetadata = null; private int version = -1; private List shardFailures = null; @@ -127,6 +131,10 @@ private void setIncludeGlobalState(Boolean includeGlobalState) { this.includeGlobalState = includeGlobalState; } + private void setUserMetadata(Map userMetadata) { + this.userMetadata = userMetadata; + } + private void setVersion(int version) { this.version = version; } @@ -153,7 +161,7 @@ public SnapshotInfo build() { } return new SnapshotInfo(snapshotId, indices, snapshotState, reason, version, startTime, endTime, - totalShards, successfulShards, shardFailures, includeGlobalState); + totalShards, successfulShards, shardFailures, includeGlobalState, userMetadata); } } @@ -194,6 +202,7 @@ int getSuccessfulShards() { SNAPSHOT_INFO_PARSER.declareLong(SnapshotInfoBuilder::setEndTime, new ParseField(END_TIME_IN_MILLIS)); SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setShardStatsBuilder, SHARD_STATS_PARSER, new ParseField(SHARDS)); SNAPSHOT_INFO_PARSER.declareBoolean(SnapshotInfoBuilder::setIncludeGlobalState, new ParseField(INCLUDE_GLOBAL_STATE)); + SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setUserMetadata, (p, c) -> p.map() , new ParseField(USER_METADATA)); SNAPSHOT_INFO_PARSER.declareInt(SnapshotInfoBuilder::setVersion, new ParseField(VERSION_ID)); SNAPSHOT_INFO_PARSER.declareObjectArray(SnapshotInfoBuilder::setShardFailures, SnapshotShardFailure.SNAPSHOT_SHARD_FAILURE_PARSER, new ParseField(FAILURES)); @@ -223,6 +232,9 @@ int getSuccessfulShards() { @Nullable private final Boolean includeGlobalState; + @Nullable + private final Map userMetadata; + @Nullable private final Version version; @@ -230,28 +242,30 @@ int getSuccessfulShards() { public SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state) { this(snapshotId, indices, state, null, null, 0L, 0L, 0, 0, - Collections.emptyList(), null); + Collections.emptyList(), null, null); } public SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state, Version version) { this(snapshotId, indices, state, null, version, 0L, 0L, 0, 0, - Collections.emptyList(), null); + Collections.emptyList(), null, null); } - public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, Boolean includeGlobalState) { + public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, Boolean includeGlobalState, + Map userMetadata) { this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, - 0, 0, Collections.emptyList(), includeGlobalState); + 0, 0, Collections.emptyList(), includeGlobalState, userMetadata); } public SnapshotInfo(SnapshotId snapshotId, List indices, long startTime, String reason, long endTime, - int totalShards, List shardFailures, Boolean includeGlobalState) { + int totalShards, List shardFailures, Boolean includeGlobalState, + Map userMetadata) { this(snapshotId, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT, - startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState); + startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures, includeGlobalState, userMetadata); } private SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState state, String reason, Version version, long startTime, long endTime, int totalShards, int successfulShards, List shardFailures, - Boolean includeGlobalState) { + Boolean includeGlobalState, Map userMetadata) { this.snapshotId = Objects.requireNonNull(snapshotId); this.indices = Collections.unmodifiableList(Objects.requireNonNull(indices)); this.state = state; @@ -263,6 +277,7 @@ private SnapshotInfo(SnapshotId snapshotId, List indices, SnapshotState this.successfulShards = successfulShards; this.shardFailures = Objects.requireNonNull(shardFailures); this.includeGlobalState = includeGlobalState; + this.userMetadata = userMetadata; } /** @@ -294,6 +309,11 @@ public SnapshotInfo(final StreamInput in) throws IOException { } version = in.readBoolean() ? Version.readVersion(in) : null; includeGlobalState = in.readOptionalBoolean(); + if (in.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + userMetadata = in.readMap(); + } else { + userMetadata = null; + } } /** @@ -304,7 +324,7 @@ public static SnapshotInfo incompatible(SnapshotId snapshotId) { return new SnapshotInfo(snapshotId, Collections.emptyList(), SnapshotState.INCOMPATIBLE, "the snapshot is incompatible with the current version of Elasticsearch and its exact version is unknown", null, 0L, 0L, 0, 0, - Collections.emptyList(), null); + Collections.emptyList(), null, null); } /** @@ -428,6 +448,15 @@ public Version version() { return version; } + /** + * Returns the custom metadata that was attached to this snapshot at creation time. + * @return custom metadata + */ + @Nullable + public Map userMetadata() { + return userMetadata; + } + /** * Compares two snapshots by their start time; if the start times are the same, then * compares the two snapshots by their snapshot ids. @@ -492,6 +521,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (includeGlobalState != null) { builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState); } + if (userMetadata != null) { + builder.field(USER_METADATA, userMetadata); + } if (verbose || state != null) { builder.field(STATE, state); } @@ -543,6 +575,7 @@ private XContentBuilder toXContentInternal(final XContentBuilder builder, final if (includeGlobalState != null) { builder.field(INCLUDE_GLOBAL_STATE, includeGlobalState); } + builder.field(USER_METADATA, userMetadata); builder.field(START_TIME, startTime); builder.field(END_TIME, endTime); builder.field(TOTAL_SHARDS, totalShards); @@ -573,6 +606,7 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr int totalShards = 0; int successfulShards = 0; Boolean includeGlobalState = null; + Map userMetadata = null; List shardFailures = Collections.emptyList(); if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); @@ -628,8 +662,12 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr parser.skipChildren(); } } else if (token == XContentParser.Token.START_OBJECT) { - // It was probably created by newer version - ignoring - parser.skipChildren(); + if (USER_METADATA.equals(currentFieldName)) { + userMetadata = parser.map(); + } else { + // It was probably created by newer version - ignoring + parser.skipChildren(); + } } } } @@ -651,7 +689,8 @@ public static SnapshotInfo fromXContentInternal(final XContentParser parser) thr totalShards, successfulShards, shardFailures, - includeGlobalState); + includeGlobalState, + userMetadata); } @Override @@ -683,6 +722,9 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeBoolean(false); } out.writeOptionalBoolean(includeGlobalState); + if (out.getVersion().onOrAfter(METADATA_FIELD_INTRODUCED)) { + out.writeMap(userMetadata); + } } private static SnapshotState snapshotState(final String reason, final List shardFailures) { @@ -712,13 +754,14 @@ public boolean equals(Object o) { Objects.equals(indices, that.indices) && Objects.equals(includeGlobalState, that.includeGlobalState) && Objects.equals(version, that.version) && - Objects.equals(shardFailures, that.shardFailures); + Objects.equals(shardFailures, that.shardFailures) && + Objects.equals(userMetadata, that.userMetadata); } @Override public int hashCode() { return Objects.hash(snapshotId, state, reason, indices, startTime, endTime, - totalShards, successfulShards, includeGlobalState, version, shardFailures); + totalShards, successfulShards, includeGlobalState, version, shardFailures, userMetadata); } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index a6138b8f6052b..2783d635c90a4 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -287,7 +287,8 @@ public ClusterState execute(ClusterState currentState) { snapshotIndices, System.currentTimeMillis(), repositoryData.getGenId(), - null); + null, + request.userMetadata()); initializingSnapshots.add(newSnapshot.snapshot()); snapshots = new SnapshotsInProgress(newSnapshot); } else { @@ -557,7 +558,8 @@ private void cleanupAfterError(Exception exception) { 0, Collections.emptyList(), snapshot.getRepositoryStateId(), - snapshot.includeGlobalState()); + snapshot.includeGlobalState(), + snapshot.userMetadata()); } catch (Exception inner) { inner.addSuppressed(exception); logger.warn(() -> new ParameterizedMessage("[{}] failed to close snapshot in repository", @@ -572,7 +574,7 @@ private void cleanupAfterError(Exception exception) { private static SnapshotInfo inProgressSnapshot(SnapshotsInProgress.Entry entry) { return new SnapshotInfo(entry.snapshot().getSnapshotId(), entry.indices().stream().map(IndexId::getName).collect(Collectors.toList()), - entry.startTime(), entry.includeGlobalState()); + entry.startTime(), entry.includeGlobalState(), entry.userMetadata()); } /** @@ -988,7 +990,8 @@ protected void doRun() { entry.shards().size(), unmodifiableList(shardFailures), entry.getRepositoryStateId(), - entry.includeGlobalState()); + entry.includeGlobalState(), + entry.userMetadata()); removeSnapshotFromClusterState(snapshot, snapshotInfo, null); logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state()); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java index 0b598be6849cb..9f7bd5f6a0149 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java @@ -19,10 +19,12 @@ package org.elasticsearch.action.admin.cluster.snapshots.create; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.IndicesOptions.Option; import org.elasticsearch.action.support.IndicesOptions.WildcardStates; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent.MapParams; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -41,6 +43,10 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.snapshots.SnapshotInfoTests.randomUserMetadata; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + public class CreateSnapshotRequestTests extends ESTestCase { // tests creating XContent and parsing with source(Map) equivalency @@ -80,6 +86,10 @@ public void testToXContent() throws IOException { original.includeGlobalState(randomBoolean()); } + if (randomBoolean()) { + original.userMetadata(randomUserMetadata()); + } + if (randomBoolean()) { Collection wildcardStates = randomSubsetOf(Arrays.asList(WildcardStates.values())); Collection