From 6bcfcb8bb779e440c7942f671240520497374225 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 18 Apr 2018 11:45:01 +0200 Subject: [PATCH 01/19] Include size of snapshot in snapshot metadata Adds difference of number of files (and file sizes) between prev and current snapshot. Total number/size reflects total number/size of files in snapshot. Closes #18543 --- .../status/SnapshotIndexShardStatus.java | 4 +- .../snapshots/status/SnapshotStats.java | 63 +++++++++-- .../snapshots/IndexShardSnapshotStatus.java | 62 ++++++++--- .../BlobStoreIndexShardSnapshot.java | 89 +++++++++++---- .../blobstore/BlobStoreRepository.java | 25 +++-- .../cluster/snapshots/SnapshotBlocksIT.java | 104 +++++++++++++++++- .../snapshots/status/SnapshotStatusTests.java | 12 +- .../index/shard/IndexShardTestCase.java | 2 +- 8 files changed, 296 insertions(+), 65 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java index 1b7ead5b96510..3a796a492e990 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java @@ -74,8 +74,8 @@ private SnapshotIndexShardStatus() { throw new IllegalArgumentException("Unknown stage type " + indexShardStatus.getStage()); } this.stats = new SnapshotStats(indexShardStatus.getStartTime(), indexShardStatus.getTotalTime(), - indexShardStatus.getNumberOfFiles(), indexShardStatus.getProcessedFiles(), - indexShardStatus.getTotalSize(), indexShardStatus.getProcessedSize()); + indexShardStatus.getDifferenceOfNumberOfFiles(), indexShardStatus.getTotalNumberOfFiles(), indexShardStatus.getProcessedFiles(), + indexShardStatus.getDifferenceOfSize(), indexShardStatus.getTotalSize(), indexShardStatus.getProcessedSize()); this.failure = indexShardStatus.getFailure(); this.nodeId = nodeId; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index 25951f73abc53..a2ec32736c5d3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.cluster.snapshots.status; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -34,19 +35,25 @@ public class SnapshotStats implements Streamable, ToXContentFragment { private long startTime; private long time; - private int numberOfFiles; + private int differenceOfNumberOfFiles; + private int totalNumberOfFiles; private int processedFiles; + private long differenceOfSize; private long totalSize; private long processedSize; SnapshotStats() { } - SnapshotStats(long startTime, long time, int numberOfFiles, int processedFiles, long totalSize, long processedSize) { + SnapshotStats(long startTime, long time, + int differenceOfNumberOfFiles, int totalNumberOfFiles, int processedFiles, + long differenceOfSize, long totalSize, long processedSize) { this.startTime = startTime; this.time = time; - this.numberOfFiles = numberOfFiles; + this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; + this.totalNumberOfFiles = totalNumberOfFiles; this.processedFiles = processedFiles; + this.differenceOfSize = differenceOfSize; this.totalSize = totalSize; this.processedSize = processedSize; } @@ -66,10 +73,17 @@ public long getTime() { } /** - * Returns number of files in the snapshot + * Returns difference of number of files between previous and this snapshot */ - public int getNumberOfFiles() { - return numberOfFiles; + public int getDifferenceOfNumberOfFiles() { + return differenceOfNumberOfFiles; + } + + /** + * Returns total number of files in the snapshot + */ + public int getTotalNumberOfFiles() { + return totalNumberOfFiles; } /** @@ -79,6 +93,13 @@ public int getProcessedFiles() { return processedFiles; } + /** + * Return difference size of files between previous and this snapshot + */ + public long getDifferenceOfSize() { + return differenceOfSize; + } + /** * Returns total size of files in the snapshot */ @@ -105,11 +126,16 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(startTime); out.writeVLong(time); - out.writeVInt(numberOfFiles); + out.writeVInt(totalNumberOfFiles); out.writeVInt(processedFiles); out.writeVLong(totalSize); out.writeVLong(processedSize); + + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeVInt(differenceOfNumberOfFiles); + out.writeVLong(differenceOfSize); + } } @Override @@ -117,17 +143,28 @@ public void readFrom(StreamInput in) throws IOException { startTime = in.readVLong(); time = in.readVLong(); - numberOfFiles = in.readVInt(); + totalNumberOfFiles = in.readVInt(); processedFiles = in.readVInt(); totalSize = in.readVLong(); processedSize = in.readVLong(); + + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + differenceOfNumberOfFiles = in.readVInt(); + differenceOfSize = in.readVLong(); + } else { + differenceOfNumberOfFiles = totalNumberOfFiles; + differenceOfSize = totalSize; + } } static final class Fields { static final String STATS = "stats"; - static final String NUMBER_OF_FILES = "number_of_files"; + static final String DIFFERENCE_OF_NUMBER_OF_FILES = "difference_of_number_of_files"; + static final String TOTAL_NUMBER_OF_FILES = "total_number_of_files"; static final String PROCESSED_FILES = "processed_files"; + static final String DIFFERENCE_OF_SIZE_IN_BYTES = "difference_of_size_in_bytes"; + static final String DIFFERENCE_OF_SIZE = "difference_of_size"; static final String TOTAL_SIZE_IN_BYTES = "total_size_in_bytes"; static final String TOTAL_SIZE = "total_size"; static final String PROCESSED_SIZE_IN_BYTES = "processed_size_in_bytes"; @@ -140,8 +177,10 @@ static final class Fields { @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(Fields.STATS); - builder.field(Fields.NUMBER_OF_FILES, getNumberOfFiles()); + builder.field(Fields.DIFFERENCE_OF_NUMBER_OF_FILES, getDifferenceOfNumberOfFiles()); + builder.field(Fields.TOTAL_NUMBER_OF_FILES, getTotalNumberOfFiles()); builder.field(Fields.PROCESSED_FILES, getProcessedFiles()); + builder.humanReadableField(Fields.DIFFERENCE_OF_SIZE_IN_BYTES, Fields.DIFFERENCE_OF_SIZE, new ByteSizeValue(getDifferenceOfSize())); builder.humanReadableField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, new ByteSizeValue(getTotalSize())); builder.humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES, Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())); builder.field(Fields.START_TIME_IN_MILLIS, getStartTime()); @@ -151,9 +190,11 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } void add(SnapshotStats stats) { - numberOfFiles += stats.numberOfFiles; + differenceOfNumberOfFiles += stats.differenceOfNumberOfFiles; + totalNumberOfFiles += stats.totalNumberOfFiles; processedFiles += stats.processedFiles; + differenceOfSize += stats.differenceOfSize; totalSize += stats.totalSize; processedSize += stats.processedSize; diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java index f1c247a41bb6d..58c3e7fea85e2 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java @@ -60,31 +60,39 @@ public enum Stage { private final AtomicReference stage; private long startTime; private long totalTime; - private int numberOfFiles; + private int differenceOfNumberOfFiles; + private int totalNumberOfFiles; private int processedFiles; private long totalSize; + private long differenceOfSize; private long processedSize; private long indexVersion; private String failure; private IndexShardSnapshotStatus(final Stage stage, final long startTime, final long totalTime, - final int numberOfFiles, final int processedFiles, final long totalSize, final long processedSize, + final int differenceOfNumberOfFiles, final int totalNumberOfFiles, final int processedFiles, + final long differenceOfSize, final long totalSize, final long processedSize, final long indexVersion, final String failure) { this.stage = new AtomicReference<>(Objects.requireNonNull(stage)); this.startTime = startTime; this.totalTime = totalTime; - this.numberOfFiles = numberOfFiles; + this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; + this.totalNumberOfFiles = totalNumberOfFiles; this.processedFiles = processedFiles; this.totalSize = totalSize; this.processedSize = processedSize; + this.differenceOfSize = differenceOfSize; this.indexVersion = indexVersion; this.failure = failure; } - public synchronized Copy moveToStarted(final long startTime, final int numberOfFiles, final long totalSize) { + public synchronized Copy moveToStarted(final long startTime, final int differenceOfNumberOfFiles, final int numberOfFiles, + final long diffSize, final long totalSize) { if (stage.compareAndSet(Stage.INIT, Stage.STARTED)) { this.startTime = startTime; - this.numberOfFiles = numberOfFiles; + this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; + this.totalNumberOfFiles = numberOfFiles; + this.differenceOfSize = diffSize; this.totalSize = totalSize; } else { throw new IllegalStateException("Unable to move the shard snapshot status to [STARTED]: " + @@ -146,12 +154,14 @@ public synchronized void addProcessedFile(long size) { * @return a {@link IndexShardSnapshotStatus.Copy} */ public synchronized IndexShardSnapshotStatus.Copy asCopy() { - return new IndexShardSnapshotStatus.Copy(stage.get(), startTime, totalTime, numberOfFiles, processedFiles, totalSize, processedSize, - indexVersion, failure); + return new IndexShardSnapshotStatus.Copy(stage.get(), startTime, totalTime, + differenceOfNumberOfFiles, totalNumberOfFiles, processedFiles, + differenceOfSize, totalSize, processedSize, + indexVersion, failure); } public static IndexShardSnapshotStatus newInitializing() { - return new IndexShardSnapshotStatus(Stage.INIT, 0L, 0L, 0, 0, 0, 0, 0, null); + return new IndexShardSnapshotStatus(Stage.INIT, 0L, 0L, 0, 0, 0, 0, 0, 0, 0, null); } public static IndexShardSnapshotStatus newFailed(final String failure) { @@ -159,12 +169,15 @@ public static IndexShardSnapshotStatus newFailed(final String failure) { if (failure == null) { throw new IllegalArgumentException("A failure description is required for a failed IndexShardSnapshotStatus"); } - return new IndexShardSnapshotStatus(Stage.FAILURE, 0L, 0L, 0, 0, 0, 0, 0, failure); + return new IndexShardSnapshotStatus(Stage.FAILURE, 0L, 0L, 0, 0, 0, 0, 0, 0, 0, failure); } - public static IndexShardSnapshotStatus newDone(final long startTime, final long totalTime, final int files, final long size) { + public static IndexShardSnapshotStatus newDone(final long startTime, final long totalTime, + final int differenceOfNumberOfFiles, final int files, + final long differenceOfSize, final long size) { // The snapshot is done which means the number of processed files is the same as total - return new IndexShardSnapshotStatus(Stage.DONE, startTime, totalTime, files, files, size, size, 0, null); + return new IndexShardSnapshotStatus(Stage.DONE, startTime, totalTime, differenceOfNumberOfFiles, files, differenceOfNumberOfFiles, + differenceOfSize, size, differenceOfSize, 0, null); } /** @@ -175,23 +188,28 @@ public static class Copy { private final Stage stage; private final long startTime; private final long totalTime; - private final int numberOfFiles; + private final int differenceOfNumberOfFiles; + private final int totalNumberOfFiles; private final int processedFiles; private final long totalSize; private final long processedSize; + private final long differenceOfSize; private final long indexVersion; private final String failure; public Copy(final Stage stage, final long startTime, final long totalTime, - final int numberOfFiles, final int processedFiles, final long totalSize, final long processedSize, + final int differenceOfNumberOfFiles, final int totalNumberOfFiles, final int processedFiles, + final long differenceOfSize, final long totalSize, final long processedSize, final long indexVersion, final String failure) { this.stage = stage; this.startTime = startTime; this.totalTime = totalTime; - this.numberOfFiles = numberOfFiles; + this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; + this.totalNumberOfFiles = totalNumberOfFiles; this.processedFiles = processedFiles; this.totalSize = totalSize; this.processedSize = processedSize; + this.differenceOfSize = differenceOfSize; this.indexVersion = indexVersion; this.failure = failure; } @@ -208,14 +226,22 @@ public long getTotalTime() { return totalTime; } - public int getNumberOfFiles() { - return numberOfFiles; + public int getDifferenceOfNumberOfFiles() { + return differenceOfNumberOfFiles; + } + + public int getTotalNumberOfFiles() { + return totalNumberOfFiles; } public int getProcessedFiles() { return processedFiles; } + public long getDifferenceOfSize() { + return differenceOfSize; + } + public long getTotalSize() { return totalSize; } @@ -238,8 +264,10 @@ public String toString() { "stage=" + stage + ", startTime=" + startTime + ", totalTime=" + totalTime + - ", numberOfFiles=" + numberOfFiles + + ", differenceOfNumberOfFiles=" + differenceOfNumberOfFiles + + ", totalNumberOfFiles=" + totalNumberOfFiles + ", processedFiles=" + processedFiles + + ", differenceOfSize=" + differenceOfSize + ", totalSize=" + totalSize + ", processedSize=" + processedSize + ", indexVersion=" + indexVersion + diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index ee285cc4f9569..84fd3ba4f4837 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -356,7 +356,11 @@ public String toString() { private final long time; - private final int numberOfFiles; + private final int differenceOfNumberOfFiles; + + private final int totalNumberOfFiles; + + private final long differenceOfSize; private final long totalSize; @@ -365,16 +369,20 @@ public String toString() { /** * Constructs new shard snapshot metadata from snapshot metadata * - * @param snapshot snapshot id - * @param indexVersion index version - * @param indexFiles list of files in the shard - * @param startTime snapshot start time - * @param time snapshot running time - * @param numberOfFiles number of files that where snapshotted - * @param totalSize total size of all files snapshotted + * @param snapshot snapshot id + * @param indexVersion index version + * @param indexFiles list of files in the shard + * @param startTime snapshot start time + * @param time snapshot running time + * @param differenceOfNumberOfFiles difference of number of files between previous and current snapshot + * @param totalNumberOfFiles total number of files that were snapshotted + * @param differenceOfSize difference between sizes of files of previous snapshot and current + * @param totalSize total size of all files snapshotted */ - public BlobStoreIndexShardSnapshot(String snapshot, long indexVersion, List indexFiles, long startTime, long time, - int numberOfFiles, long totalSize) { + public BlobStoreIndexShardSnapshot(String snapshot, long indexVersion, List indexFiles, + long startTime, long time, + int differenceOfNumberOfFiles, int totalNumberOfFiles, + long differenceOfSize, long totalSize) { assert snapshot != null; assert indexVersion >= 0; this.snapshot = snapshot; @@ -382,7 +390,9 @@ public BlobStoreIndexShardSnapshot(String snapshot, long indexVersion, List(indexFiles)); this.startTime = startTime; this.time = time; - this.numberOfFiles = numberOfFiles; + this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; + this.totalNumberOfFiles = totalNumberOfFiles; + this.differenceOfSize = differenceOfSize; this.totalSize = totalSize; } @@ -395,7 +405,9 @@ private BlobStoreIndexShardSnapshot() { this.indexFiles = Collections.emptyList(); this.startTime = 0; this.time = 0; - this.numberOfFiles = 0; + this.differenceOfNumberOfFiles = 0; + this.totalNumberOfFiles = 0; + this.differenceOfSize = 0; this.totalSize = 0; } @@ -440,11 +452,25 @@ public long time() { return time; } + /** + * Returns difference of number of files between previously snapshotted and current one + */ + public int differenceOfNumberOfFiles() { + return differenceOfNumberOfFiles; + } + /** * Returns number of files that where snapshotted */ public int numberOfFiles() { - return numberOfFiles; + return totalNumberOfFiles; + } + + /** + * Returns difference of size of files between previous and actual snapshot + */ + public long differenceOfSize() { + return differenceOfSize; } /** @@ -458,7 +484,9 @@ public long totalSize() { private static final String INDEX_VERSION = "index_version"; private static final String START_TIME = "start_time"; private static final String TIME = "time"; - private static final String NUMBER_OF_FILES = "number_of_files"; + private static final String DIFFERENCE_OF_NUMBER_OF_FILES = "difference_of_number_of_files"; + private static final String TOTAL_NUMBER_OF_FILES = "total_number_of_files"; + private static final String DIFFERENCE_OF_SIZE = "difference_of_size"; private static final String TOTAL_SIZE = "total_size"; private static final String FILES = "files"; @@ -466,9 +494,11 @@ public long totalSize() { private static final ParseField PARSE_INDEX_VERSION = new ParseField("index_version", "index-version"); private static final ParseField PARSE_START_TIME = new ParseField("start_time"); private static final ParseField PARSE_TIME = new ParseField("time"); - private static final ParseField PARSE_NUMBER_OF_FILES = new ParseField("number_of_files"); - private static final ParseField PARSE_TOTAL_SIZE = new ParseField("total_size"); - private static final ParseField PARSE_FILES = new ParseField("files"); + private static final ParseField PARSE_DIFFERENCE_OF_NUMBER_OF_FILES = new ParseField(DIFFERENCE_OF_NUMBER_OF_FILES, "number_of_files"); + private static final ParseField PARSE_TOTAL_NUMBER_OF_FILES = new ParseField(TOTAL_NUMBER_OF_FILES); + private static final ParseField PARSE_DIFFERENCE_OF_SIZE = new ParseField(DIFFERENCE_OF_SIZE); + private static final ParseField PARSE_TOTAL_SIZE = new ParseField(TOTAL_SIZE); + private static final ParseField PARSE_FILES = new ParseField(FILES); /** * Serializes shard snapshot metadata info into JSON @@ -482,7 +512,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(INDEX_VERSION, indexVersion); builder.field(START_TIME, startTime); builder.field(TIME, time); - builder.field(NUMBER_OF_FILES, numberOfFiles); + builder.field(DIFFERENCE_OF_NUMBER_OF_FILES, differenceOfNumberOfFiles); + builder.field(TOTAL_NUMBER_OF_FILES, totalNumberOfFiles); + builder.field(DIFFERENCE_OF_SIZE, differenceOfSize); builder.field(TOTAL_SIZE, totalSize); builder.startArray(FILES); for (FileInfo fileInfo : indexFiles) { @@ -503,7 +535,9 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th long indexVersion = -1; long startTime = 0; long time = 0; - int numberOfFiles = 0; + int differenceOfNumberOfFiles = 0; + int totalNumberOfFiles = -1; + long differenceOfSize = -1; long totalSize = 0; List indexFiles = new ArrayList<>(); @@ -526,8 +560,12 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th startTime = parser.longValue(); } else if (PARSE_TIME.match(currentFieldName, parser.getDeprecationHandler())) { time = parser.longValue(); - } else if (PARSE_NUMBER_OF_FILES.match(currentFieldName, parser.getDeprecationHandler())) { - numberOfFiles = parser.intValue(); + } else if (PARSE_DIFFERENCE_OF_NUMBER_OF_FILES.match(currentFieldName, parser.getDeprecationHandler())) { + differenceOfNumberOfFiles = parser.intValue(); + } else if (PARSE_TOTAL_NUMBER_OF_FILES.match(currentFieldName, parser.getDeprecationHandler())) { + totalNumberOfFiles = parser.intValue(); + } else if (PARSE_DIFFERENCE_OF_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { + differenceOfSize = parser.longValue(); } else if (PARSE_TOTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { totalSize = parser.longValue(); } else { @@ -549,7 +587,14 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th } } } + // for the sake of backward compatibility + if (totalNumberOfFiles < 0){ + totalNumberOfFiles = differenceOfNumberOfFiles; + } + if (differenceOfSize < 0){ + differenceOfSize = totalSize; + } return new BlobStoreIndexShardSnapshot(snapshot, indexVersion, Collections.unmodifiableList(indexFiles), - startTime, time, numberOfFiles, totalSize); + startTime, time, differenceOfNumberOfFiles, totalNumberOfFiles, differenceOfSize, totalSize); } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index f1adf9273ffde..f6b8c7da69bf2 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -817,7 +817,9 @@ public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version versio public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) { Context context = new Context(snapshotId, version, indexId, shardId); BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot(); - return IndexShardSnapshotStatus.newDone(snapshot.startTime(), snapshot.time(), snapshot.numberOfFiles(), snapshot.totalSize()); + return IndexShardSnapshotStatus.newDone(snapshot.startTime(), snapshot.time(), + snapshot.differenceOfNumberOfFiles(), snapshot.numberOfFiles(), + snapshot.differenceOfSize(), snapshot.totalSize()); } @Override @@ -1116,9 +1118,11 @@ public void snapshot(final IndexCommit snapshotIndexCommit) { final List indexCommitPointFiles = new ArrayList<>(); store.incRef(); + int indexDifferenceOfNumberOfFiles = 0; + int indexTotalNumberOfFiles = 0; + long indexDiffFilesSize = 0; + long indexTotalFilesSize = 0; try { - int indexNumberOfFiles = 0; - long indexTotalFilesSize = 0; ArrayList filesToSnapshot = new ArrayList<>(); final Store.MetadataSnapshot metadata; // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should @@ -1158,9 +1162,13 @@ public void snapshot(final IndexCommit snapshotIndexCommit) { } } } + + indexTotalFilesSize += md.length(); + indexTotalNumberOfFiles++; + if (existingFileInfo == null) { - indexNumberOfFiles++; - indexTotalFilesSize += md.length(); + indexDifferenceOfNumberOfFiles++; + indexDiffFilesSize += md.length(); // create a new FileInfo BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = new BlobStoreIndexShardSnapshot.FileInfo(fileNameFromGeneration(++generation), md, chunkSize()); indexCommitPointFiles.add(snapshotFileInfo); @@ -1170,7 +1178,8 @@ public void snapshot(final IndexCommit snapshotIndexCommit) { } } - snapshotStatus.moveToStarted(startTime, indexNumberOfFiles, indexTotalFilesSize); + snapshotStatus.moveToStarted(startTime, indexDifferenceOfNumberOfFiles, + indexTotalNumberOfFiles, indexDiffFilesSize, indexTotalFilesSize); for (BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo : filesToSnapshot) { try { @@ -1193,7 +1202,9 @@ public void snapshot(final IndexCommit snapshotIndexCommit) { // snapshotStatus.startTime() is assigned on the same machine, // so it's safe to use with VLong System.currentTimeMillis() - lastSnapshotStatus.getStartTime(), - lastSnapshotStatus.getNumberOfFiles(), + lastSnapshotStatus.getDifferenceOfNumberOfFiles(), + lastSnapshotStatus.getTotalNumberOfFiles(), + lastSnapshotStatus.getDifferenceOfSize(), lastSnapshotStatus.getTotalSize()); //TODO: The time stored in snapshot doesn't include cleanup time. diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index c66fa4b244f18..70f00c5275466 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -22,6 +22,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats; +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -31,12 +33,23 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.junit.Before; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; /** * This class tests that snapshot operations (Create, Delete, Restore) are blocked when the cluster is read-only. @@ -52,6 +65,8 @@ public class SnapshotBlocksIT extends ESIntegTestCase { protected static final String REPOSITORY_NAME = "repo-" + INDEX_NAME; protected static final String SNAPSHOT_NAME = "snapshot-0"; + private Path repoPath; + @Before protected void setUpRepository() throws Exception { createIndex(INDEX_NAME, OTHER_INDEX_NAME); @@ -65,11 +80,12 @@ protected void setUpRepository() throws Exception { client().prepareIndex(OTHER_INDEX_NAME, "type").setSource("test", "init").execute().actionGet(); } - logger.info("--> register a repository"); + + repoPath = randomRepoPath(); assertAcked(client().admin().cluster().preparePutRepository(REPOSITORY_NAME) .setType("fs") - .setSettings(Settings.builder().put("location", randomRepoPath()))); + .setSettings(Settings.builder().put("location", repoPath))); logger.info("--> verify the repository"); VerifyRepositoryResponse verifyResponse = client().admin().cluster().prepareVerifyRepository(REPOSITORY_NAME).get(); @@ -84,6 +100,90 @@ protected void setUpRepository() throws Exception { ensureSearchable(); } + public void testSnapshotTotalAndDifferenceSizes() throws IOException { + SnapshotsStatusResponse response = client().admin().cluster().prepareSnapshotStatus(REPOSITORY_NAME) + .setSnapshots(SNAPSHOT_NAME) + .execute() + .actionGet(); + + List snapshots = response.getSnapshots(); + + List files = scanSnapshotFolder(); + assertThat(snapshots, hasSize(1)); + SnapshotStats stats = snapshots.get(0).getStats(); + + + assertThat(stats.getTotalNumberOfFiles(), is(files.size())); + assertThat(stats.getTotalSize(), is(files.stream().mapToLong(f -> { + try { + return Files.size(f); + } catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + }).sum())); + + assertThat(stats.getDifferenceOfNumberOfFiles(), equalTo(stats.getProcessedFiles())); + assertThat(stats.getDifferenceOfSize(), equalTo(stats.getProcessedSize())); + + // add few docs - less than initially + int docs = between(1, 5); + for (int i = 0; i < docs; i++) { + client().prepareIndex(INDEX_NAME, "type").setSource("test", "test" + i).execute().actionGet(); + } + + // create another snapshot and drop 1st one + // total size has to grow, and has to be equal to files on fs + assertThat(client().admin().cluster() + .prepareCreateSnapshot(REPOSITORY_NAME, "snapshot-1") + .setWaitForCompletion(true).get().status(), + equalTo(RestStatus.OK)); + + assertTrue(client().admin().cluster() + .prepareDeleteSnapshot(REPOSITORY_NAME, SNAPSHOT_NAME) + .get().isAcknowledged()); + + response = client().admin().cluster().prepareSnapshotStatus(REPOSITORY_NAME) + .setSnapshots("snapshot-1") + .execute() + .actionGet(); + + final List files1 = scanSnapshotFolder(); + + snapshots = response.getSnapshots(); + + SnapshotStats anotherStats = snapshots.get(0).getStats(); + + assertThat(anotherStats.getDifferenceOfNumberOfFiles(), equalTo(anotherStats.getProcessedFiles())); + assertThat(anotherStats.getDifferenceOfSize(), equalTo(anotherStats.getProcessedSize())); + + assertThat(stats.getTotalSize(), lessThan(anotherStats.getTotalSize())); + assertThat(stats.getTotalNumberOfFiles(), lessThan(anotherStats.getTotalNumberOfFiles())); + + assertThat(anotherStats.getTotalNumberOfFiles(), is(files1.size())); + assertThat(anotherStats.getTotalSize(), is(files1.stream().mapToLong(f -> { + try { + return Files.size(f); + } catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + }).sum())); + } + + private List scanSnapshotFolder() throws IOException { + List files = new ArrayList<>(); + Files.walkFileTree(repoPath, new SimpleFileVisitor(){ + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.getFileName().toString().startsWith("__")){ + files.add(file); + } + return super.visitFile(file, attrs); + } + } + ); + return files; + } + public void testCreateSnapshotWithBlocks() { logger.info("--> creating a snapshot is allowed when the cluster is read only"); try { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java index 5c38617461072..6a34af41f87c8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java @@ -91,8 +91,10 @@ public void testToString() throws Exception { " \"total\" : " + totalShards + "\n" + " },\n" + " \"stats\" : {\n" + - " \"number_of_files\" : 0,\n" + + " \"difference_of_number_of_files\" : 0,\n" + + " \"total_number_of_files\" : 0,\n" + " \"processed_files\" : 0,\n" + + " \"difference_of_size_in_bytes\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + @@ -109,8 +111,10 @@ public void testToString() throws Exception { " \"total\" : " + totalShards + "\n" + " },\n" + " \"stats\" : {\n" + - " \"number_of_files\" : 0,\n" + + " \"difference_of_number_of_files\" : 0,\n" + + " \"total_number_of_files\" : 0,\n" + " \"processed_files\" : 0,\n" + + " \"difference_of_size_in_bytes\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + @@ -120,8 +124,10 @@ public void testToString() throws Exception { " \"" + shardId + "\" : {\n" + " \"stage\" : \"" + shardStage.toString() + "\",\n" + " \"stats\" : {\n" + - " \"number_of_files\" : 0,\n" + + " \"difference_of_number_of_files\" : 0,\n" + + " \"total_number_of_files\" : 0,\n" + " \"processed_files\" : 0,\n" + + " \"difference_of_size_in_bytes\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index a0e1cfc334110..bd9592f94bc28 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -634,7 +634,7 @@ protected void snapshotShard(final IndexShard shard, final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy(); assertEquals(IndexShardSnapshotStatus.Stage.DONE, lastSnapshotStatus.getStage()); - assertEquals(shard.snapshotStoreMetadata().size(), lastSnapshotStatus.getNumberOfFiles()); + assertEquals(shard.snapshotStoreMetadata().size(), lastSnapshotStatus.getTotalNumberOfFiles()); assertNull(lastSnapshotStatus.getFailure()); } From a08c8077d6cadbe3372b560178bb82078b55430e Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Thu, 19 Apr 2018 15:54:31 +0200 Subject: [PATCH 02/19] Include size of snapshot in snapshot metadata - changes on Yannick's PR --- .../status/SnapshotIndexShardStatus.java | 4 +- .../snapshots/status/SnapshotStats.java | 87 ++++++------ .../snapshots/IndexShardSnapshotStatus.java | 88 ++++++------ .../BlobStoreIndexShardSnapshot.java | 111 +++++++-------- .../blobstore/BlobStoreRepository.java | 27 ++-- .../cluster/snapshots/SnapshotBlocksIT.java | 102 +------------- .../snapshots/status/SnapshotStatusTests.java | 24 ++-- .../DedicatedClusterSnapshotRestoreIT.java | 126 ++++++++++++++++++ .../SharedClusterSnapshotRestoreIT.java | 6 +- .../index/shard/IndexShardTestCase.java | 2 +- 10 files changed, 295 insertions(+), 282 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java index 3a796a492e990..39abd8613caa4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java @@ -74,8 +74,8 @@ private SnapshotIndexShardStatus() { throw new IllegalArgumentException("Unknown stage type " + indexShardStatus.getStage()); } this.stats = new SnapshotStats(indexShardStatus.getStartTime(), indexShardStatus.getTotalTime(), - indexShardStatus.getDifferenceOfNumberOfFiles(), indexShardStatus.getTotalNumberOfFiles(), indexShardStatus.getProcessedFiles(), - indexShardStatus.getDifferenceOfSize(), indexShardStatus.getTotalSize(), indexShardStatus.getProcessedSize()); + indexShardStatus.getIncrementalFileCount(), indexShardStatus.getTotalFileCount(), indexShardStatus.getProcessedFileCount(), + indexShardStatus.getIncrementalSize(), indexShardStatus.getTotalSize(), indexShardStatus.getProcessedSize()); this.failure = indexShardStatus.getFailure(); this.nodeId = nodeId; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index a2ec32736c5d3..1e2442b41d732 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -35,10 +35,10 @@ public class SnapshotStats implements Streamable, ToXContentFragment { private long startTime; private long time; - private int differenceOfNumberOfFiles; - private int totalNumberOfFiles; - private int processedFiles; - private long differenceOfSize; + private int incrementalFileCount; + private int totalFileCount; + private int processedFileCount; + private long incrementalSize; private long totalSize; private long processedSize; @@ -46,14 +46,14 @@ public class SnapshotStats implements Streamable, ToXContentFragment { } SnapshotStats(long startTime, long time, - int differenceOfNumberOfFiles, int totalNumberOfFiles, int processedFiles, - long differenceOfSize, long totalSize, long processedSize) { + int incrementalFileCount, int totalFileCount, int processedFileCount, + long incrementalSize, long totalSize, long processedSize) { this.startTime = startTime; this.time = time; - this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; - this.totalNumberOfFiles = totalNumberOfFiles; - this.processedFiles = processedFiles; - this.differenceOfSize = differenceOfSize; + this.incrementalFileCount = incrementalFileCount; + this.totalFileCount = totalFileCount; + this.processedFileCount = processedFileCount; + this.incrementalSize = incrementalSize; this.totalSize = totalSize; this.processedSize = processedSize; } @@ -73,31 +73,31 @@ public long getTime() { } /** - * Returns difference of number of files between previous and this snapshot + * Returns incremental file count of the snapshot */ - public int getDifferenceOfNumberOfFiles() { - return differenceOfNumberOfFiles; + public int getIncrementalFileCount() { + return incrementalFileCount; } /** * Returns total number of files in the snapshot */ - public int getTotalNumberOfFiles() { - return totalNumberOfFiles; + public int getTotalFileCount() { + return totalFileCount; } /** * Returns number of files in the snapshot that were processed so far */ - public int getProcessedFiles() { - return processedFiles; + public int getProcessedFileCount() { + return processedFileCount; } /** - * Return difference size of files between previous and this snapshot + * Return incremental files size of the snapshot */ - public long getDifferenceOfSize() { - return differenceOfSize; + public long getIncrementalSize() { + return incrementalSize; } /** @@ -126,15 +126,15 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(startTime); out.writeVLong(time); - out.writeVInt(totalNumberOfFiles); - out.writeVInt(processedFiles); + out.writeVInt(totalFileCount); + out.writeVInt(processedFileCount); out.writeVLong(totalSize); out.writeVLong(processedSize); if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { - out.writeVInt(differenceOfNumberOfFiles); - out.writeVLong(differenceOfSize); + out.writeVInt(incrementalFileCount); + out.writeVLong(incrementalSize); } } @@ -143,28 +143,28 @@ public void readFrom(StreamInput in) throws IOException { startTime = in.readVLong(); time = in.readVLong(); - totalNumberOfFiles = in.readVInt(); - processedFiles = in.readVInt(); + totalFileCount = in.readVInt(); + processedFileCount = in.readVInt(); totalSize = in.readVLong(); processedSize = in.readVLong(); if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { - differenceOfNumberOfFiles = in.readVInt(); - differenceOfSize = in.readVLong(); + incrementalFileCount = in.readVInt(); + incrementalSize = in.readVLong(); } else { - differenceOfNumberOfFiles = totalNumberOfFiles; - differenceOfSize = totalSize; + incrementalFileCount = totalFileCount; + incrementalSize = totalSize; } } static final class Fields { static final String STATS = "stats"; - static final String DIFFERENCE_OF_NUMBER_OF_FILES = "difference_of_number_of_files"; - static final String TOTAL_NUMBER_OF_FILES = "total_number_of_files"; - static final String PROCESSED_FILES = "processed_files"; - static final String DIFFERENCE_OF_SIZE_IN_BYTES = "difference_of_size_in_bytes"; - static final String DIFFERENCE_OF_SIZE = "difference_of_size"; + static final String INCREMENTAL_FILE_COUNT = "incremental_file_count"; + static final String TOTAL_FILE_COUNT = "total_file_count"; + static final String PROCESSED_FILE_COUNT = "processed_file_count"; + static final String INCREMENTAL_SIZE_IN_BYTES = "incremental_size_in_bytes"; + static final String INCREMENTAL_SIZE = "incremental_size"; static final String TOTAL_SIZE_IN_BYTES = "total_size_in_bytes"; static final String TOTAL_SIZE = "total_size"; static final String PROCESSED_SIZE_IN_BYTES = "processed_size_in_bytes"; @@ -177,10 +177,10 @@ static final class Fields { @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(Fields.STATS); - builder.field(Fields.DIFFERENCE_OF_NUMBER_OF_FILES, getDifferenceOfNumberOfFiles()); - builder.field(Fields.TOTAL_NUMBER_OF_FILES, getTotalNumberOfFiles()); - builder.field(Fields.PROCESSED_FILES, getProcessedFiles()); - builder.humanReadableField(Fields.DIFFERENCE_OF_SIZE_IN_BYTES, Fields.DIFFERENCE_OF_SIZE, new ByteSizeValue(getDifferenceOfSize())); + builder.field(Fields.INCREMENTAL_FILE_COUNT, getIncrementalFileCount()); + builder.field(Fields.TOTAL_FILE_COUNT, getTotalFileCount()); + builder.field(Fields.PROCESSED_FILE_COUNT, getProcessedFileCount()); + builder.humanReadableField(Fields.INCREMENTAL_SIZE_IN_BYTES, Fields.INCREMENTAL_SIZE, new ByteSizeValue(getIncrementalSize())); builder.humanReadableField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, new ByteSizeValue(getTotalSize())); builder.humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES, Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())); builder.field(Fields.START_TIME_IN_MILLIS, getStartTime()); @@ -190,15 +190,14 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } void add(SnapshotStats stats) { - differenceOfNumberOfFiles += stats.differenceOfNumberOfFiles; - totalNumberOfFiles += stats.totalNumberOfFiles; - processedFiles += stats.processedFiles; + incrementalFileCount += stats.incrementalFileCount; + totalFileCount += stats.totalFileCount; + processedFileCount += stats.processedFileCount; - differenceOfSize += stats.differenceOfSize; + incrementalSize += stats.incrementalSize; totalSize += stats.totalSize; processedSize += stats.processedSize; - if (startTime == 0) { // First time here startTime = stats.startTime; diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java index 58c3e7fea85e2..bfc3faae9344f 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java @@ -60,39 +60,39 @@ public enum Stage { private final AtomicReference stage; private long startTime; private long totalTime; - private int differenceOfNumberOfFiles; - private int totalNumberOfFiles; - private int processedFiles; + private int incrementalFileCount; + private int totalFileCount; + private int processedFileCount; private long totalSize; - private long differenceOfSize; + private long incrementalSize; private long processedSize; private long indexVersion; private String failure; private IndexShardSnapshotStatus(final Stage stage, final long startTime, final long totalTime, - final int differenceOfNumberOfFiles, final int totalNumberOfFiles, final int processedFiles, - final long differenceOfSize, final long totalSize, final long processedSize, + final int incrementalFileCount, final int totalFileCount, final int processedFileCount, + final long incrementalSize, final long totalSize, final long processedSize, final long indexVersion, final String failure) { this.stage = new AtomicReference<>(Objects.requireNonNull(stage)); this.startTime = startTime; this.totalTime = totalTime; - this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; - this.totalNumberOfFiles = totalNumberOfFiles; - this.processedFiles = processedFiles; + this.incrementalFileCount = incrementalFileCount; + this.totalFileCount = totalFileCount; + this.processedFileCount = processedFileCount; this.totalSize = totalSize; this.processedSize = processedSize; - this.differenceOfSize = differenceOfSize; + this.incrementalSize = incrementalSize; this.indexVersion = indexVersion; this.failure = failure; } - public synchronized Copy moveToStarted(final long startTime, final int differenceOfNumberOfFiles, final int numberOfFiles, - final long diffSize, final long totalSize) { + public synchronized Copy moveToStarted(final long startTime, final int incrementalFileCount, final int totalFileCount, + final long incrementalSize, final long totalSize) { if (stage.compareAndSet(Stage.INIT, Stage.STARTED)) { this.startTime = startTime; - this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; - this.totalNumberOfFiles = numberOfFiles; - this.differenceOfSize = diffSize; + this.incrementalFileCount = incrementalFileCount; + this.totalFileCount = totalFileCount; + this.incrementalSize = incrementalSize; this.totalSize = totalSize; } else { throw new IllegalStateException("Unable to move the shard snapshot status to [STARTED]: " + @@ -143,7 +143,7 @@ public boolean isAborted() { * Increments number of processed files */ public synchronized void addProcessedFile(long size) { - processedFiles++; + processedFileCount++; processedSize += size; } @@ -155,8 +155,8 @@ public synchronized void addProcessedFile(long size) { */ public synchronized IndexShardSnapshotStatus.Copy asCopy() { return new IndexShardSnapshotStatus.Copy(stage.get(), startTime, totalTime, - differenceOfNumberOfFiles, totalNumberOfFiles, processedFiles, - differenceOfSize, totalSize, processedSize, + incrementalFileCount, totalFileCount, processedFileCount, + incrementalSize, totalSize, processedSize, indexVersion, failure); } @@ -173,11 +173,11 @@ public static IndexShardSnapshotStatus newFailed(final String failure) { } public static IndexShardSnapshotStatus newDone(final long startTime, final long totalTime, - final int differenceOfNumberOfFiles, final int files, - final long differenceOfSize, final long size) { + final int incrementalFileCount, final int fileCount, + final long incrementalSize, final long size) { // The snapshot is done which means the number of processed files is the same as total - return new IndexShardSnapshotStatus(Stage.DONE, startTime, totalTime, differenceOfNumberOfFiles, files, differenceOfNumberOfFiles, - differenceOfSize, size, differenceOfSize, 0, null); + return new IndexShardSnapshotStatus(Stage.DONE, startTime, totalTime, incrementalFileCount, fileCount, incrementalFileCount, + incrementalSize, size, incrementalSize, 0, null); } /** @@ -188,28 +188,28 @@ public static class Copy { private final Stage stage; private final long startTime; private final long totalTime; - private final int differenceOfNumberOfFiles; - private final int totalNumberOfFiles; - private final int processedFiles; + private final int incrementalFileCount; + private final int totalFileCount; + private final int processedFileCount; private final long totalSize; private final long processedSize; - private final long differenceOfSize; + private final long incrementalSize; private final long indexVersion; private final String failure; public Copy(final Stage stage, final long startTime, final long totalTime, - final int differenceOfNumberOfFiles, final int totalNumberOfFiles, final int processedFiles, - final long differenceOfSize, final long totalSize, final long processedSize, + final int incrementalFileCount, final int totalFileCount, final int processedFileCount, + final long incrementalSize, final long totalSize, final long processedSize, final long indexVersion, final String failure) { this.stage = stage; this.startTime = startTime; this.totalTime = totalTime; - this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; - this.totalNumberOfFiles = totalNumberOfFiles; - this.processedFiles = processedFiles; + this.incrementalFileCount = incrementalFileCount; + this.totalFileCount = totalFileCount; + this.processedFileCount = processedFileCount; this.totalSize = totalSize; this.processedSize = processedSize; - this.differenceOfSize = differenceOfSize; + this.incrementalSize = incrementalSize; this.indexVersion = indexVersion; this.failure = failure; } @@ -226,20 +226,20 @@ public long getTotalTime() { return totalTime; } - public int getDifferenceOfNumberOfFiles() { - return differenceOfNumberOfFiles; + public int getIncrementalFileCount() { + return incrementalFileCount; } - public int getTotalNumberOfFiles() { - return totalNumberOfFiles; + public int getTotalFileCount() { + return totalFileCount; } - public int getProcessedFiles() { - return processedFiles; + public int getProcessedFileCount() { + return processedFileCount; } - public long getDifferenceOfSize() { - return differenceOfSize; + public long getIncrementalSize() { + return incrementalSize; } public long getTotalSize() { @@ -264,10 +264,10 @@ public String toString() { "stage=" + stage + ", startTime=" + startTime + ", totalTime=" + totalTime + - ", differenceOfNumberOfFiles=" + differenceOfNumberOfFiles + - ", totalNumberOfFiles=" + totalNumberOfFiles + - ", processedFiles=" + processedFiles + - ", differenceOfSize=" + differenceOfSize + + ", incrementalFileCount=" + incrementalFileCount + + ", totalFileCount=" + totalFileCount + + ", processedFileCount=" + processedFileCount + + ", incrementalSize=" + incrementalSize + ", totalSize=" + totalSize + ", processedSize=" + processedSize + ", indexVersion=" + indexVersion + diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 84fd3ba4f4837..4ab1be9d823bc 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -356,11 +356,11 @@ public String toString() { private final long time; - private final int differenceOfNumberOfFiles; + private final int incrementalFileCount; - private final int totalNumberOfFiles; + private final int totalFileCount; - private final long differenceOfSize; + private final long incrementalSize; private final long totalSize; @@ -369,20 +369,19 @@ public String toString() { /** * Constructs new shard snapshot metadata from snapshot metadata * - * @param snapshot snapshot id - * @param indexVersion index version - * @param indexFiles list of files in the shard - * @param startTime snapshot start time - * @param time snapshot running time - * @param differenceOfNumberOfFiles difference of number of files between previous and current snapshot - * @param totalNumberOfFiles total number of files that were snapshotted - * @param differenceOfSize difference between sizes of files of previous snapshot and current - * @param totalSize total size of all files snapshotted + * @param snapshot snapshot id + * @param indexVersion index version + * @param indexFiles list of files in the shard + * @param startTime snapshot start time + * @param time snapshot running time + * @param incrementalFileCount incremental of files that were snapshotted + * @param incrementalSize incremental size of snapshot */ public BlobStoreIndexShardSnapshot(String snapshot, long indexVersion, List indexFiles, long startTime, long time, - int differenceOfNumberOfFiles, int totalNumberOfFiles, - long differenceOfSize, long totalSize) { + int incrementalFileCount, + long incrementalSize + ) { assert snapshot != null; assert indexVersion >= 0; this.snapshot = snapshot; @@ -390,9 +389,17 @@ public BlobStoreIndexShardSnapshot(String snapshot, long indexVersion, List(indexFiles)); this.startTime = startTime; this.time = time; - this.differenceOfNumberOfFiles = differenceOfNumberOfFiles; - this.totalNumberOfFiles = totalNumberOfFiles; - this.differenceOfSize = differenceOfSize; + this.incrementalFileCount = incrementalFileCount; + this.incrementalSize = incrementalSize; + + int totalFileCount = 0; + long totalSize = 0; + for (FileInfo indexFile : indexFiles) { + totalFileCount ++; + totalSize += indexFile.metadata.length(); + } + + this.totalFileCount = totalFileCount; this.totalSize = totalSize; } @@ -405,9 +412,9 @@ private BlobStoreIndexShardSnapshot() { this.indexFiles = Collections.emptyList(); this.startTime = 0; this.time = 0; - this.differenceOfNumberOfFiles = 0; - this.totalNumberOfFiles = 0; - this.differenceOfSize = 0; + this.incrementalFileCount = 0; + this.totalFileCount = 0; + this.incrementalSize = 0; this.totalSize = 0; } @@ -453,24 +460,24 @@ public long time() { } /** - * Returns difference of number of files between previously snapshotted and current one + * Returns incremental of files that were snapshotted */ - public int differenceOfNumberOfFiles() { - return differenceOfNumberOfFiles; + public int incrementalFileCount() { + return incrementalFileCount; } /** - * Returns number of files that where snapshotted + * Returns total number of files that where snapshotted */ - public int numberOfFiles() { - return totalNumberOfFiles; + public int totalFileCount() { + return totalFileCount; } /** - * Returns difference of size of files between previous and actual snapshot + * Returns incremental of files size that were snapshotted */ - public long differenceOfSize() { - return differenceOfSize; + public long incrementalSize() { + return incrementalSize; } /** @@ -484,20 +491,16 @@ public long totalSize() { private static final String INDEX_VERSION = "index_version"; private static final String START_TIME = "start_time"; private static final String TIME = "time"; - private static final String DIFFERENCE_OF_NUMBER_OF_FILES = "difference_of_number_of_files"; - private static final String TOTAL_NUMBER_OF_FILES = "total_number_of_files"; - private static final String DIFFERENCE_OF_SIZE = "difference_of_size"; - private static final String TOTAL_SIZE = "total_size"; + private static final String INCREMENTAL_FILE_COUNT = "incremental_file_count"; + private static final String INCREMENTAL_SIZE = "incremental_size"; private static final String FILES = "files"; private static final ParseField PARSE_NAME = new ParseField("name"); private static final ParseField PARSE_INDEX_VERSION = new ParseField("index_version", "index-version"); private static final ParseField PARSE_START_TIME = new ParseField("start_time"); private static final ParseField PARSE_TIME = new ParseField("time"); - private static final ParseField PARSE_DIFFERENCE_OF_NUMBER_OF_FILES = new ParseField(DIFFERENCE_OF_NUMBER_OF_FILES, "number_of_files"); - private static final ParseField PARSE_TOTAL_NUMBER_OF_FILES = new ParseField(TOTAL_NUMBER_OF_FILES); - private static final ParseField PARSE_DIFFERENCE_OF_SIZE = new ParseField(DIFFERENCE_OF_SIZE); - private static final ParseField PARSE_TOTAL_SIZE = new ParseField(TOTAL_SIZE); + private static final ParseField PARSE_INCREMENTAL_FILE_COUNT = new ParseField(INCREMENTAL_FILE_COUNT, "number_of_files"); + private static final ParseField PARSE_INCREMENTAL_SIZE = new ParseField(INCREMENTAL_SIZE, "total_size"); private static final ParseField PARSE_FILES = new ParseField(FILES); /** @@ -512,10 +515,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(INDEX_VERSION, indexVersion); builder.field(START_TIME, startTime); builder.field(TIME, time); - builder.field(DIFFERENCE_OF_NUMBER_OF_FILES, differenceOfNumberOfFiles); - builder.field(TOTAL_NUMBER_OF_FILES, totalNumberOfFiles); - builder.field(DIFFERENCE_OF_SIZE, differenceOfSize); - builder.field(TOTAL_SIZE, totalSize); + builder.field(INCREMENTAL_FILE_COUNT, incrementalFileCount); + builder.field(INCREMENTAL_SIZE, incrementalSize); builder.startArray(FILES); for (FileInfo fileInfo : indexFiles) { FileInfo.toXContent(fileInfo, builder, params); @@ -535,10 +536,8 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th long indexVersion = -1; long startTime = 0; long time = 0; - int differenceOfNumberOfFiles = 0; - int totalNumberOfFiles = -1; - long differenceOfSize = -1; - long totalSize = 0; + int incrementalFileCount = 0; + long incrementalSize = 0; List indexFiles = new ArrayList<>(); if (parser.currentToken() == null) { // fresh parser? move to the first token @@ -560,14 +559,10 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th startTime = parser.longValue(); } else if (PARSE_TIME.match(currentFieldName, parser.getDeprecationHandler())) { time = parser.longValue(); - } else if (PARSE_DIFFERENCE_OF_NUMBER_OF_FILES.match(currentFieldName, parser.getDeprecationHandler())) { - differenceOfNumberOfFiles = parser.intValue(); - } else if (PARSE_TOTAL_NUMBER_OF_FILES.match(currentFieldName, parser.getDeprecationHandler())) { - totalNumberOfFiles = parser.intValue(); - } else if (PARSE_DIFFERENCE_OF_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { - differenceOfSize = parser.longValue(); - } else if (PARSE_TOTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { - totalSize = parser.longValue(); + } else if (PARSE_INCREMENTAL_FILE_COUNT.match(currentFieldName, parser.getDeprecationHandler())) { + incrementalFileCount = parser.intValue(); + } else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { + incrementalSize = parser.longValue(); } else { throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); } @@ -587,14 +582,8 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th } } } - // for the sake of backward compatibility - if (totalNumberOfFiles < 0){ - totalNumberOfFiles = differenceOfNumberOfFiles; - } - if (differenceOfSize < 0){ - differenceOfSize = totalSize; - } + return new BlobStoreIndexShardSnapshot(snapshot, indexVersion, Collections.unmodifiableList(indexFiles), - startTime, time, differenceOfNumberOfFiles, totalNumberOfFiles, differenceOfSize, totalSize); + startTime, time, incrementalFileCount, incrementalSize); } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index f6b8c7da69bf2..d1fc172b8d825 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -818,8 +818,8 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Ve Context context = new Context(snapshotId, version, indexId, shardId); BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot(); return IndexShardSnapshotStatus.newDone(snapshot.startTime(), snapshot.time(), - snapshot.differenceOfNumberOfFiles(), snapshot.numberOfFiles(), - snapshot.differenceOfSize(), snapshot.totalSize()); + snapshot.incrementalFileCount(), snapshot.totalFileCount(), + snapshot.incrementalSize(), snapshot.totalSize()); } @Override @@ -1118,10 +1118,10 @@ public void snapshot(final IndexCommit snapshotIndexCommit) { final List indexCommitPointFiles = new ArrayList<>(); store.incRef(); - int indexDifferenceOfNumberOfFiles = 0; + int indexIncrementalFileCount = 0; int indexTotalNumberOfFiles = 0; - long indexDiffFilesSize = 0; - long indexTotalFilesSize = 0; + long indexIncrementalSize = 0; + long indexTotalFileCount = 0; try { ArrayList filesToSnapshot = new ArrayList<>(); final Store.MetadataSnapshot metadata; @@ -1163,12 +1163,12 @@ public void snapshot(final IndexCommit snapshotIndexCommit) { } } - indexTotalFilesSize += md.length(); + indexTotalFileCount += md.length(); indexTotalNumberOfFiles++; if (existingFileInfo == null) { - indexDifferenceOfNumberOfFiles++; - indexDiffFilesSize += md.length(); + indexIncrementalFileCount++; + indexIncrementalSize += md.length(); // create a new FileInfo BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = new BlobStoreIndexShardSnapshot.FileInfo(fileNameFromGeneration(++generation), md, chunkSize()); indexCommitPointFiles.add(snapshotFileInfo); @@ -1178,8 +1178,8 @@ public void snapshot(final IndexCommit snapshotIndexCommit) { } } - snapshotStatus.moveToStarted(startTime, indexDifferenceOfNumberOfFiles, - indexTotalNumberOfFiles, indexDiffFilesSize, indexTotalFilesSize); + snapshotStatus.moveToStarted(startTime, indexIncrementalFileCount, + indexTotalNumberOfFiles, indexIncrementalSize, indexTotalFileCount); for (BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo : filesToSnapshot) { try { @@ -1202,10 +1202,9 @@ public void snapshot(final IndexCommit snapshotIndexCommit) { // snapshotStatus.startTime() is assigned on the same machine, // so it's safe to use with VLong System.currentTimeMillis() - lastSnapshotStatus.getStartTime(), - lastSnapshotStatus.getDifferenceOfNumberOfFiles(), - lastSnapshotStatus.getTotalNumberOfFiles(), - lastSnapshotStatus.getDifferenceOfSize(), - lastSnapshotStatus.getTotalSize()); + lastSnapshotStatus.getIncrementalFileCount(), + lastSnapshotStatus.getIncrementalSize() + ); //TODO: The time stored in snapshot doesn't include cleanup time. logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index 70f00c5275466..85c0d24163778 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -22,8 +22,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats; -import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -33,23 +31,12 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.junit.Before; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.List; - import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThan; /** * This class tests that snapshot operations (Create, Delete, Restore) are blocked when the cluster is read-only. @@ -65,8 +52,6 @@ public class SnapshotBlocksIT extends ESIntegTestCase { protected static final String REPOSITORY_NAME = "repo-" + INDEX_NAME; protected static final String SNAPSHOT_NAME = "snapshot-0"; - private Path repoPath; - @Before protected void setUpRepository() throws Exception { createIndex(INDEX_NAME, OTHER_INDEX_NAME); @@ -82,10 +67,9 @@ protected void setUpRepository() throws Exception { logger.info("--> register a repository"); - repoPath = randomRepoPath(); assertAcked(client().admin().cluster().preparePutRepository(REPOSITORY_NAME) .setType("fs") - .setSettings(Settings.builder().put("location", repoPath))); + .setSettings(Settings.builder().put("location", randomRepoPath()))); logger.info("--> verify the repository"); VerifyRepositoryResponse verifyResponse = client().admin().cluster().prepareVerifyRepository(REPOSITORY_NAME).get(); @@ -100,90 +84,6 @@ protected void setUpRepository() throws Exception { ensureSearchable(); } - public void testSnapshotTotalAndDifferenceSizes() throws IOException { - SnapshotsStatusResponse response = client().admin().cluster().prepareSnapshotStatus(REPOSITORY_NAME) - .setSnapshots(SNAPSHOT_NAME) - .execute() - .actionGet(); - - List snapshots = response.getSnapshots(); - - List files = scanSnapshotFolder(); - assertThat(snapshots, hasSize(1)); - SnapshotStats stats = snapshots.get(0).getStats(); - - - assertThat(stats.getTotalNumberOfFiles(), is(files.size())); - assertThat(stats.getTotalSize(), is(files.stream().mapToLong(f -> { - try { - return Files.size(f); - } catch (IOException e) { - throw new RuntimeException(e.getMessage(), e); - } - }).sum())); - - assertThat(stats.getDifferenceOfNumberOfFiles(), equalTo(stats.getProcessedFiles())); - assertThat(stats.getDifferenceOfSize(), equalTo(stats.getProcessedSize())); - - // add few docs - less than initially - int docs = between(1, 5); - for (int i = 0; i < docs; i++) { - client().prepareIndex(INDEX_NAME, "type").setSource("test", "test" + i).execute().actionGet(); - } - - // create another snapshot and drop 1st one - // total size has to grow, and has to be equal to files on fs - assertThat(client().admin().cluster() - .prepareCreateSnapshot(REPOSITORY_NAME, "snapshot-1") - .setWaitForCompletion(true).get().status(), - equalTo(RestStatus.OK)); - - assertTrue(client().admin().cluster() - .prepareDeleteSnapshot(REPOSITORY_NAME, SNAPSHOT_NAME) - .get().isAcknowledged()); - - response = client().admin().cluster().prepareSnapshotStatus(REPOSITORY_NAME) - .setSnapshots("snapshot-1") - .execute() - .actionGet(); - - final List files1 = scanSnapshotFolder(); - - snapshots = response.getSnapshots(); - - SnapshotStats anotherStats = snapshots.get(0).getStats(); - - assertThat(anotherStats.getDifferenceOfNumberOfFiles(), equalTo(anotherStats.getProcessedFiles())); - assertThat(anotherStats.getDifferenceOfSize(), equalTo(anotherStats.getProcessedSize())); - - assertThat(stats.getTotalSize(), lessThan(anotherStats.getTotalSize())); - assertThat(stats.getTotalNumberOfFiles(), lessThan(anotherStats.getTotalNumberOfFiles())); - - assertThat(anotherStats.getTotalNumberOfFiles(), is(files1.size())); - assertThat(anotherStats.getTotalSize(), is(files1.stream().mapToLong(f -> { - try { - return Files.size(f); - } catch (IOException e) { - throw new RuntimeException(e.getMessage(), e); - } - }).sum())); - } - - private List scanSnapshotFolder() throws IOException { - List files = new ArrayList<>(); - Files.walkFileTree(repoPath, new SimpleFileVisitor(){ - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (file.getFileName().toString().startsWith("__")){ - files.add(file); - } - return super.visitFile(file, attrs); - } - } - ); - return files; - } - public void testCreateSnapshotWithBlocks() { logger.info("--> creating a snapshot is allowed when the cluster is read only"); try { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java index 6a34af41f87c8..328cdc478dddc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java @@ -91,10 +91,10 @@ public void testToString() throws Exception { " \"total\" : " + totalShards + "\n" + " },\n" + " \"stats\" : {\n" + - " \"difference_of_number_of_files\" : 0,\n" + - " \"total_number_of_files\" : 0,\n" + - " \"processed_files\" : 0,\n" + - " \"difference_of_size_in_bytes\" : 0,\n" + + " \"incremental_file_count\" : 0,\n" + + " \"total_file_count\" : 0,\n" + + " \"processed_file_count\" : 0,\n" + + " \"incremental_size_in_bytes\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + @@ -111,10 +111,10 @@ public void testToString() throws Exception { " \"total\" : " + totalShards + "\n" + " },\n" + " \"stats\" : {\n" + - " \"difference_of_number_of_files\" : 0,\n" + - " \"total_number_of_files\" : 0,\n" + - " \"processed_files\" : 0,\n" + - " \"difference_of_size_in_bytes\" : 0,\n" + + " \"incremental_file_count\" : 0,\n" + + " \"total_file_count\" : 0,\n" + + " \"processed_file_count\" : 0,\n" + + " \"incremental_size_in_bytes\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + @@ -124,10 +124,10 @@ public void testToString() throws Exception { " \"" + shardId + "\" : {\n" + " \"stage\" : \"" + shardStage.toString() + "\",\n" + " \"stats\" : {\n" + - " \"difference_of_number_of_files\" : 0,\n" + - " \"total_number_of_files\" : 0,\n" + - " \"processed_files\" : 0,\n" + - " \"difference_of_size_in_bytes\" : 0,\n" + + " \"incremental_file_count\" : 0,\n" + + " \"total_file_count\" : 0,\n" + + " \"processed_file_count\" : 0,\n" + + " \"incremental_size_in_bytes\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 5341b268544e7..396e6403a0d85 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -23,10 +23,12 @@ import com.carrotsearch.hppc.IntSet; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; +import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -68,6 +70,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.admin.cluster.RestClusterStateAction; import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.elasticsearch.snapshots.mockstore.MockRepository; @@ -78,7 +81,11 @@ import org.elasticsearch.test.rest.FakeRestRequest; import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -96,6 +103,8 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -981,6 +990,123 @@ public void testRestoreShrinkIndex() throws Exception { ensureYellow(); } + public void testSnapshotTotalAndIncrementalSizes() throws IOException { + Client client = client(); + final String indexName = "test-blocks-1"; + final String repositoryName = "repo-" + indexName; + final String snapshot0 = "snapshot-0"; + final String snapshot1 = "snapshot-1"; + + createIndex(indexName); + + int docs = between(10, 100); + for (int i = 0; i < docs; i++) { + client.prepareIndex(indexName, "type").setSource("test", "init").execute().actionGet(); + } + + logger.info("--> register a repository"); + + final Path repoPath = randomRepoPath(); + assertAcked(client.admin().cluster().preparePutRepository(repositoryName) + .setType("fs") + .setSettings(Settings.builder().put("location", repoPath))); + + logger.info("--> verify the repository"); + VerifyRepositoryResponse verifyResponse = client.admin().cluster().prepareVerifyRepository(repositoryName).get(); + assertThat(verifyResponse.getNodes().length, equalTo(cluster().numDataAndMasterNodes())); + + logger.info("--> create a snapshot"); + CreateSnapshotResponse snapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot0) + .setIncludeGlobalState(true) + .setWaitForCompletion(true) + .execute().actionGet(); + assertThat(snapshotResponse.status(), equalTo(RestStatus.OK)); + ensureSearchable(); + + SnapshotsStatusResponse response = client.admin().cluster().prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshot0) + .execute() + .actionGet(); + + List snapshots = response.getSnapshots(); + + List files = scanSnapshotFolder(repoPath); + assertThat(snapshots, hasSize(1)); + SnapshotStats stats = snapshots.get(0).getStats(); + + + assertThat(stats.getTotalFileCount(), is(files.size())); + assertThat(stats.getTotalSize(), is(files.stream().mapToLong(f -> { + try { + return Files.size(f); + } catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + }).sum())); + + assertThat(stats.getIncrementalFileCount(), equalTo(stats.getProcessedFileCount())); + assertThat(stats.getIncrementalSize(), equalTo(stats.getProcessedSize())); + + // add few docs - less than initially + docs = between(1, 5); + for (int i = 0; i < docs; i++) { + client.prepareIndex(indexName, "type").setSource("test", "test" + i).execute().actionGet(); + } + + // create another snapshot and drop 1st one + // total size has to grow, and has to be equal to files on fs + assertThat(client.admin().cluster() + .prepareCreateSnapshot(repositoryName, snapshot1) + .setWaitForCompletion(true).get().status(), + equalTo(RestStatus.OK)); + + assertTrue(client.admin().cluster() + .prepareDeleteSnapshot(repositoryName, snapshot0) + .get().isAcknowledged()); + + response = client.admin().cluster().prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshot1) + .execute() + .actionGet(); + + final List files1 = scanSnapshotFolder(repoPath); + + snapshots = response.getSnapshots(); + + SnapshotStats anotherStats = snapshots.get(0).getStats(); + + assertThat(anotherStats.getIncrementalFileCount(), equalTo(anotherStats.getProcessedFileCount())); + assertThat(anotherStats.getIncrementalSize(), equalTo(anotherStats.getProcessedSize())); + + assertThat(stats.getTotalSize(), lessThan(anotherStats.getTotalSize())); + assertThat(stats.getTotalFileCount(), lessThan(anotherStats.getTotalFileCount())); + + assertThat(anotherStats.getTotalFileCount(), is(files1.size())); + assertThat(anotherStats.getTotalSize(), is(files1.stream().mapToLong(f -> { + try { + return Files.size(f); + } catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + }).sum())); + } + + + private List scanSnapshotFolder(Path repoPath) throws IOException { + List files = new ArrayList<>(); + Files.walkFileTree(repoPath, new SimpleFileVisitor(){ + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.getFileName().toString().startsWith("__")){ + files.add(file); + } + return super.visitFile(file, attrs); + } + } + ); + return files; + } + public static class SnapshottableMetadata extends TestCustomMetaData { public static final String TYPE = "test_snapshottable"; diff --git a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index dbaf26c965749..9d254ff3269f9 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -2047,7 +2047,7 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test").get().getSnapshots().get(0); List shards = snapshotStatus.getShards(); for (SnapshotIndexShardStatus status : shards) { - assertThat(status.getStats().getProcessedFiles(), greaterThan(1)); + assertThat(status.getStats().getProcessedFileCount(), greaterThan(1)); } } @@ -2059,7 +2059,7 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-1").get().getSnapshots().get(0); List shards = snapshotStatus.getShards(); for (SnapshotIndexShardStatus status : shards) { - assertThat(status.getStats().getProcessedFiles(), equalTo(0)); + assertThat(status.getStats().getProcessedFileCount(), equalTo(0)); } } @@ -2072,7 +2072,7 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-2").get().getSnapshots().get(0); List shards = snapshotStatus.getShards(); for (SnapshotIndexShardStatus status : shards) { - assertThat(status.getStats().getProcessedFiles(), equalTo(2)); // we flush before the snapshot such that we have to process the segments_N files plus the .del file + assertThat(status.getStats().getProcessedFileCount(), equalTo(2)); // we flush before the snapshot such that we have to process the segments_N files plus the .del file } } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index bd9592f94bc28..0bdebdd5573a6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -634,7 +634,7 @@ protected void snapshotShard(final IndexShard shard, final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.asCopy(); assertEquals(IndexShardSnapshotStatus.Stage.DONE, lastSnapshotStatus.getStage()); - assertEquals(shard.snapshotStoreMetadata().size(), lastSnapshotStatus.getTotalNumberOfFiles()); + assertEquals(shard.snapshotStoreMetadata().size(), lastSnapshotStatus.getTotalFileCount()); assertNull(lastSnapshotStatus.getFailure()); } From 50d7c789d1fe3e7e2095bf78bd8552227ac5623f Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 20 Apr 2018 13:23:11 +0200 Subject: [PATCH 03/19] Include size of snapshot in snapshot metadata - changes on 2nd Yannick's PR --- .../snapshots/status/SnapshotStats.java | 20 +++--- .../BlobStoreIndexShardSnapshot.java | 33 +++------- .../DedicatedClusterSnapshotRestoreIT.java | 63 ++++++++++--------- 3 files changed, 53 insertions(+), 63 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index 1e2442b41d732..2ac5fcf48438a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -126,15 +126,15 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(startTime); out.writeVLong(time); - out.writeVInt(totalFileCount); + out.writeVInt(incrementalFileCount); out.writeVInt(processedFileCount); - out.writeVLong(totalSize); + out.writeVLong(incrementalSize); out.writeVLong(processedSize); if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { - out.writeVInt(incrementalFileCount); - out.writeVLong(incrementalSize); + out.writeVInt(totalFileCount ); + out.writeVLong(totalSize); } } @@ -143,18 +143,18 @@ public void readFrom(StreamInput in) throws IOException { startTime = in.readVLong(); time = in.readVLong(); - totalFileCount = in.readVInt(); + incrementalFileCount = in.readVInt(); processedFileCount = in.readVInt(); - totalSize = in.readVLong(); + incrementalSize = in.readVLong(); processedSize = in.readVLong(); if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { - incrementalFileCount = in.readVInt(); - incrementalSize = in.readVLong(); + totalFileCount = in.readVInt(); + totalSize = in.readVLong(); } else { - incrementalFileCount = totalFileCount; - incrementalSize = totalSize; + totalFileCount = incrementalFileCount; + totalSize = incrementalSize; } } diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 4ab1be9d823bc..36c6f63dd927f 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -358,12 +358,8 @@ public String toString() { private final int incrementalFileCount; - private final int totalFileCount; - private final long incrementalSize; - private final long totalSize; - private final List indexFiles; /** @@ -391,16 +387,6 @@ public BlobStoreIndexShardSnapshot(String snapshot, long indexVersion, List fi.metadata().length()).sum(); } private static final String NAME = "name"; private static final String INDEX_VERSION = "index_version"; private static final String START_TIME = "start_time"; private static final String TIME = "time"; - private static final String INCREMENTAL_FILE_COUNT = "incremental_file_count"; - private static final String INCREMENTAL_SIZE = "incremental_size"; private static final String FILES = "files"; + // for the sake of BWC keep the actual property names as in 6.x + // + there is a constraint in #fromXContent() that leads to ElasticsearchParseException("unknown parameter [incremental_file_count]"); + private static final String INCREMENTAL_FILE_COUNT = "number_of_files"; + private static final String INCREMENTAL_SIZE = "total_size"; + private static final ParseField PARSE_NAME = new ParseField("name"); private static final ParseField PARSE_INDEX_VERSION = new ParseField("index_version", "index-version"); private static final ParseField PARSE_START_TIME = new ParseField("start_time"); private static final ParseField PARSE_TIME = new ParseField("time"); - private static final ParseField PARSE_INCREMENTAL_FILE_COUNT = new ParseField(INCREMENTAL_FILE_COUNT, "number_of_files"); - private static final ParseField PARSE_INCREMENTAL_SIZE = new ParseField(INCREMENTAL_SIZE, "total_size"); + private static final ParseField PARSE_INCREMENTAL_FILE_COUNT = new ParseField(INCREMENTAL_FILE_COUNT); + private static final ParseField PARSE_INCREMENTAL_SIZE = new ParseField(INCREMENTAL_SIZE); private static final ParseField PARSE_FILES = new ParseField(FILES); /** diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 396e6403a0d85..35319dc33ef6b 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -81,6 +81,7 @@ import org.elasticsearch.test.rest.FakeRestRequest; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -1011,38 +1012,37 @@ public void testSnapshotTotalAndIncrementalSizes() throws IOException { .setType("fs") .setSettings(Settings.builder().put("location", repoPath))); - logger.info("--> verify the repository"); - VerifyRepositoryResponse verifyResponse = client.admin().cluster().prepareVerifyRepository(repositoryName).get(); - assertThat(verifyResponse.getNodes().length, equalTo(cluster().numDataAndMasterNodes())); - logger.info("--> create a snapshot"); - CreateSnapshotResponse snapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot0) + client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot0) .setIncludeGlobalState(true) .setWaitForCompletion(true) - .execute().actionGet(); - assertThat(snapshotResponse.status(), equalTo(RestStatus.OK)); - ensureSearchable(); + .get(); SnapshotsStatusResponse response = client.admin().cluster().prepareSnapshotStatus(repositoryName) .setSnapshots(snapshot0) - .execute() - .actionGet(); + .get(); List snapshots = response.getSnapshots(); - List files = scanSnapshotFolder(repoPath); + List snapshot0Files = scanSnapshotFolder(repoPath); assertThat(snapshots, hasSize(1)); - SnapshotStats stats = snapshots.get(0).getStats(); - - assertThat(stats.getTotalFileCount(), is(files.size())); - assertThat(stats.getTotalSize(), is(files.stream().mapToLong(f -> { + final int snapshot0FileCount = snapshot0Files.size(); + final long snapshot0FileSize = snapshot0Files.stream().mapToLong(f -> { try { return Files.size(f); } catch (IOException e) { - throw new RuntimeException(e.getMessage(), e); + throw new UncheckedIOException(e); } - }).sum())); + }).sum(); + + SnapshotStats stats = snapshots.get(0).getStats(); + + assertThat(stats.getTotalFileCount(), is(snapshot0FileCount)); + assertThat(stats.getTotalSize(), is(snapshot0FileSize)); + + assertThat(stats.getIncrementalFileCount(), equalTo(snapshot0FileCount)); + assertThat(stats.getIncrementalSize(), equalTo(snapshot0FileSize)); assertThat(stats.getIncrementalFileCount(), equalTo(stats.getProcessedFileCount())); assertThat(stats.getIncrementalSize(), equalTo(stats.getProcessedSize())); @@ -1053,23 +1053,32 @@ public void testSnapshotTotalAndIncrementalSizes() throws IOException { client.prepareIndex(indexName, "type").setSource("test", "test" + i).execute().actionGet(); } - // create another snapshot and drop 1st one - // total size has to grow, and has to be equal to files on fs + // create another snapshot + // total size has to grow and has to be equal to files on fs assertThat(client.admin().cluster() .prepareCreateSnapshot(repositoryName, snapshot1) .setWaitForCompletion(true).get().status(), equalTo(RestStatus.OK)); + // drop 1st one to avoid miscalculation as snapshot reuses some files of prev snapshot assertTrue(client.admin().cluster() .prepareDeleteSnapshot(repositoryName, snapshot0) .get().isAcknowledged()); response = client.admin().cluster().prepareSnapshotStatus(repositoryName) .setSnapshots(snapshot1) - .execute() - .actionGet(); + .get(); + + final List snapshot1Files = scanSnapshotFolder(repoPath); - final List files1 = scanSnapshotFolder(repoPath); + final int snapshot1FileCount = snapshot1Files.size(); + final long snapshot1FileSize = snapshot1Files.stream().mapToLong(f -> { + try { + return Files.size(f); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).sum(); snapshots = response.getSnapshots(); @@ -1081,14 +1090,8 @@ public void testSnapshotTotalAndIncrementalSizes() throws IOException { assertThat(stats.getTotalSize(), lessThan(anotherStats.getTotalSize())); assertThat(stats.getTotalFileCount(), lessThan(anotherStats.getTotalFileCount())); - assertThat(anotherStats.getTotalFileCount(), is(files1.size())); - assertThat(anotherStats.getTotalSize(), is(files1.stream().mapToLong(f -> { - try { - return Files.size(f); - } catch (IOException e) { - throw new RuntimeException(e.getMessage(), e); - } - }).sum())); + assertThat(anotherStats.getTotalFileCount(), is(snapshot1FileCount)); + assertThat(anotherStats.getTotalSize(), is(snapshot1FileSize)); } From 6ea7e01c9cf6aeb01795d6baed5baff246968c0e Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 20 Apr 2018 15:09:28 +0200 Subject: [PATCH 04/19] Include size of snapshot in snapshot metadata - changes on 3nd Yannick's PR --- .../DedicatedClusterSnapshotRestoreIT.java | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 35319dc33ef6b..df142b617677e 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -1028,13 +1028,7 @@ public void testSnapshotTotalAndIncrementalSizes() throws IOException { assertThat(snapshots, hasSize(1)); final int snapshot0FileCount = snapshot0Files.size(); - final long snapshot0FileSize = snapshot0Files.stream().mapToLong(f -> { - try { - return Files.size(f); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }).sum(); + final long snapshot0FileSize = calculateTotalFilesSize(snapshot0Files); SnapshotStats stats = snapshots.get(0).getStats(); @@ -1072,18 +1066,18 @@ public void testSnapshotTotalAndIncrementalSizes() throws IOException { final List snapshot1Files = scanSnapshotFolder(repoPath); final int snapshot1FileCount = snapshot1Files.size(); - final long snapshot1FileSize = snapshot1Files.stream().mapToLong(f -> { - try { - return Files.size(f); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }).sum(); + final long snapshot1FileSize = calculateTotalFilesSize(snapshot1Files); snapshots = response.getSnapshots(); SnapshotStats anotherStats = snapshots.get(0).getStats(); + ArrayList snapshotFilesDiff = new ArrayList<>(snapshot1Files); + snapshotFilesDiff.removeAll(snapshot0Files); + + assertThat(anotherStats.getIncrementalFileCount(), equalTo(snapshotFilesDiff.size())); + assertThat(anotherStats.getIncrementalSize(), equalTo(calculateTotalFilesSize(snapshotFilesDiff))); + assertThat(anotherStats.getIncrementalFileCount(), equalTo(anotherStats.getProcessedFileCount())); assertThat(anotherStats.getIncrementalSize(), equalTo(anotherStats.getProcessedSize())); @@ -1094,6 +1088,16 @@ public void testSnapshotTotalAndIncrementalSizes() throws IOException { assertThat(anotherStats.getTotalSize(), is(snapshot1FileSize)); } + private long calculateTotalFilesSize(List files) { + return files.stream().mapToLong(f -> { + try { + return Files.size(f); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).sum(); + } + private List scanSnapshotFolder(Path repoPath) throws IOException { List files = new ArrayList<>(); From 4ca0ba9e6b881a22013b7b152da8a1b854ec14a0 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 11 May 2018 12:46:11 +0200 Subject: [PATCH 05/19] Include size of snapshot in snapshot metadata #29602 added nested objects "incremental_files" and "all_files" + keep bwc part --- .../snapshots/status/SnapshotStats.java | 61 +++++++++++++------ .../snapshots/status/SnapshotStatusTests.java | 54 ++++++++++++---- 2 files changed, 84 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index 2ac5fcf48438a..8facba44692b4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -160,33 +160,56 @@ public void readFrom(StreamInput in) throws IOException { static final class Fields { static final String STATS = "stats"; - static final String INCREMENTAL_FILE_COUNT = "incremental_file_count"; - static final String TOTAL_FILE_COUNT = "total_file_count"; - static final String PROCESSED_FILE_COUNT = "processed_file_count"; - static final String INCREMENTAL_SIZE_IN_BYTES = "incremental_size_in_bytes"; - static final String INCREMENTAL_SIZE = "incremental_size"; - static final String TOTAL_SIZE_IN_BYTES = "total_size_in_bytes"; - static final String TOTAL_SIZE = "total_size"; - static final String PROCESSED_SIZE_IN_BYTES = "processed_size_in_bytes"; + + static final String INCREMENTAL_FILES = "incremental_files"; + static final String COUNT = "count"; + static final String SIZE = "size"; + static final String SIZE_IN_BYTES = "size_in_bytes"; + static final String PROCESSED = "processed"; static final String PROCESSED_SIZE = "processed_size"; + static final String PROCESSED_SIZE_IN_BYTES = "processed_size_in_bytes"; static final String START_TIME_IN_MILLIS = "start_time_in_millis"; static final String TIME_IN_MILLIS = "time_in_millis"; static final String TIME = "time"; + + static final String ALL_FILES = "all_files"; + + // BWC + static final String NUMBER_OF_FILES = "number_of_files"; + static final String PROCESSED_FILES = "processed_files"; + static final String TOTAL_SIZE = "total_size"; + static final String TOTAL_SIZE_IN_BYTES = "total_size_in_bytes"; + } @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(Fields.STATS); - builder.field(Fields.INCREMENTAL_FILE_COUNT, getIncrementalFileCount()); - builder.field(Fields.TOTAL_FILE_COUNT, getTotalFileCount()); - builder.field(Fields.PROCESSED_FILE_COUNT, getProcessedFileCount()); - builder.humanReadableField(Fields.INCREMENTAL_SIZE_IN_BYTES, Fields.INCREMENTAL_SIZE, new ByteSizeValue(getIncrementalSize())); - builder.humanReadableField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, new ByteSizeValue(getTotalSize())); - builder.humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES, Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())); - builder.field(Fields.START_TIME_IN_MILLIS, getStartTime()); - builder.humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())); - builder.endObject(); - return builder; + return builder.startObject(Fields.STATS) + // incremental_files starts + .startObject(Fields.INCREMENTAL_FILES) + .field(Fields.COUNT, getIncrementalFileCount()) + .humanReadableField(Fields.SIZE_IN_BYTES,Fields.SIZE, new ByteSizeValue(getIncrementalSize())) + .field(Fields.PROCESSED, getProcessedFileCount()) + .humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES,Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())) + .field(Fields.START_TIME_IN_MILLIS, getStartTime()) + .humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())) + // incremental_files ends + .endObject() + // all_files starts + .startObject(Fields.ALL_FILES) + .field(Fields.COUNT, getTotalFileCount()) + .humanReadableField(Fields.SIZE_IN_BYTES,Fields.SIZE, new ByteSizeValue(getTotalSize())) + // all_files ends + .endObject() + // BWC part + .field(Fields.NUMBER_OF_FILES, getIncrementalFileCount()) + .field(Fields.PROCESSED_FILES, getProcessedFileCount()) + .humanReadableField(Fields.TOTAL_SIZE_IN_BYTES,Fields.TOTAL_SIZE, new ByteSizeValue(getIncrementalSize())) + .humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES,Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())) + .field(Fields.START_TIME_IN_MILLIS, getStartTime()) + .humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())) + // stats ends + .endObject(); } void add(SnapshotStats stats) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java index 328cdc478dddc..a795c740850a6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java @@ -91,10 +91,20 @@ public void testToString() throws Exception { " \"total\" : " + totalShards + "\n" + " },\n" + " \"stats\" : {\n" + - " \"incremental_file_count\" : 0,\n" + - " \"total_file_count\" : 0,\n" + - " \"processed_file_count\" : 0,\n" + - " \"incremental_size_in_bytes\" : 0,\n" + + " \"incremental_files\" : {\n" + + " \"count\" : 0,\n" + + " \"size_in_bytes\" : 0,\n" + + " \"processed\" : 0,\n" + + " \"processed_size_in_bytes\" : 0,\n" + + " \"start_time_in_millis\" : 0,\n" + + " \"time_in_millis\" : 0\n" + + " },\n" + + " \"all_files\" : {\n" + + " \"count\" : 0,\n" + + " \"size_in_bytes\" : 0\n" + + " },\n" + + " \"number_of_files\" : 0,\n" + + " \"processed_files\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + @@ -111,10 +121,20 @@ public void testToString() throws Exception { " \"total\" : " + totalShards + "\n" + " },\n" + " \"stats\" : {\n" + - " \"incremental_file_count\" : 0,\n" + - " \"total_file_count\" : 0,\n" + - " \"processed_file_count\" : 0,\n" + - " \"incremental_size_in_bytes\" : 0,\n" + + " \"incremental_files\" : {\n" + + " \"count\" : 0,\n" + + " \"size_in_bytes\" : 0,\n" + + " \"processed\" : 0,\n" + + " \"processed_size_in_bytes\" : 0,\n" + + " \"start_time_in_millis\" : 0,\n" + + " \"time_in_millis\" : 0\n" + + " },\n" + + " \"all_files\" : {\n" + + " \"count\" : 0,\n" + + " \"size_in_bytes\" : 0\n" + + " },\n" + + " \"number_of_files\" : 0,\n" + + " \"processed_files\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + @@ -124,10 +144,20 @@ public void testToString() throws Exception { " \"" + shardId + "\" : {\n" + " \"stage\" : \"" + shardStage.toString() + "\",\n" + " \"stats\" : {\n" + - " \"incremental_file_count\" : 0,\n" + - " \"total_file_count\" : 0,\n" + - " \"processed_file_count\" : 0,\n" + - " \"incremental_size_in_bytes\" : 0,\n" + + " \"incremental_files\" : {\n" + + " \"count\" : 0,\n" + + " \"size_in_bytes\" : 0,\n" + + " \"processed\" : 0,\n" + + " \"processed_size_in_bytes\" : 0,\n" + + " \"start_time_in_millis\" : 0,\n" + + " \"time_in_millis\" : 0\n" + + " },\n" + + " \"all_files\" : {\n" + + " \"count\" : 0,\n" + + " \"size_in_bytes\" : 0\n" + + " },\n" + + " \"number_of_files\" : 0,\n" + + " \"processed_files\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + " \"processed_size_in_bytes\" : 0,\n" + " \"start_time_in_millis\" : 0,\n" + From 7ab7262af30027a3de7bdff066153e8aa775e0ba Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 11 May 2018 12:53:24 +0200 Subject: [PATCH 06/19] Include size of snapshot in snapshot metadata #29602 added consistent names for incremental snapshot + keep bwc part of BlobStoreIndexShardSnapshot --- .../BlobStoreIndexShardSnapshot.java | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 36c6f63dd927f..2a22e481eae54 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -476,16 +476,16 @@ public long totalSize() { private static final String START_TIME = "start_time"; private static final String TIME = "time"; private static final String FILES = "files"; + private static final String INCREMENTAL_FILE_COUNT = "incremental_file_count"; + private static final String INCREMENTAL_SIZE = "incremental_size"; // for the sake of BWC keep the actual property names as in 6.x - // + there is a constraint in #fromXContent() that leads to ElasticsearchParseException("unknown parameter [incremental_file_count]"); - private static final String INCREMENTAL_FILE_COUNT = "number_of_files"; - private static final String INCREMENTAL_SIZE = "total_size"; + private static final String NUMBER_OF_FILES = "number_of_files"; + private static final String TOTAL_SIZE = "total_size"; - - private static final ParseField PARSE_NAME = new ParseField("name"); - private static final ParseField PARSE_INDEX_VERSION = new ParseField("index_version", "index-version"); - private static final ParseField PARSE_START_TIME = new ParseField("start_time"); - private static final ParseField PARSE_TIME = new ParseField("time"); + private static final ParseField PARSE_NAME = new ParseField(NAME); + private static final ParseField PARSE_INDEX_VERSION = new ParseField(INDEX_VERSION, "index-version"); + private static final ParseField PARSE_START_TIME = new ParseField(START_TIME); + private static final ParseField PARSE_TIME = new ParseField(TIME); private static final ParseField PARSE_INCREMENTAL_FILE_COUNT = new ParseField(INCREMENTAL_FILE_COUNT); private static final ParseField PARSE_INCREMENTAL_SIZE = new ParseField(INCREMENTAL_SIZE); private static final ParseField PARSE_FILES = new ParseField(FILES); @@ -498,17 +498,20 @@ public long totalSize() { */ @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(NAME, snapshot); - builder.field(INDEX_VERSION, indexVersion); - builder.field(START_TIME, startTime); - builder.field(TIME, time); - builder.field(INCREMENTAL_FILE_COUNT, incrementalFileCount); - builder.field(INCREMENTAL_SIZE, incrementalSize); - builder.startArray(FILES); + builder.field(NAME, snapshot) + .field(INDEX_VERSION, indexVersion) + .field(START_TIME, startTime) + .field(TIME, time) + .field(INCREMENTAL_FILE_COUNT, incrementalFileCount) + .field(INCREMENTAL_SIZE, incrementalSize) + .startArray(FILES); for (FileInfo fileInfo : indexFiles) { FileInfo.toXContent(fileInfo, builder, params); } - builder.endArray(); + builder.endArray() + // BWC part + .field(NUMBER_OF_FILES, incrementalFileCount) + .field(TOTAL_SIZE, incrementalSize); return builder; } @@ -550,8 +553,6 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th incrementalFileCount = parser.intValue(); } else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { incrementalSize = parser.longValue(); - } else { - throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); } } else if (token == XContentParser.Token.START_ARRAY) { if (PARSE_FILES.match(currentFieldName, parser.getDeprecationHandler())) { From 9a839f319dcc11280f14f5e0d5732bd5b62bc888 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 11 May 2018 18:04:06 +0200 Subject: [PATCH 07/19] Revert "Include size of snapshot in snapshot metadata #29602" This reverts commit 7ab7262 --- .../BlobStoreIndexShardSnapshot.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 2a22e481eae54..275bc432942d3 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -476,11 +476,11 @@ public long totalSize() { private static final String START_TIME = "start_time"; private static final String TIME = "time"; private static final String FILES = "files"; - private static final String INCREMENTAL_FILE_COUNT = "incremental_file_count"; - private static final String INCREMENTAL_SIZE = "incremental_size"; // for the sake of BWC keep the actual property names as in 6.x - private static final String NUMBER_OF_FILES = "number_of_files"; - private static final String TOTAL_SIZE = "total_size"; + // + there is a constraint in #fromXContent() that leads to ElasticsearchParseException("unknown parameter [incremental_file_count]"); + private static final String INCREMENTAL_FILE_COUNT = "number_of_files"; + private static final String INCREMENTAL_SIZE = "total_size"; + private static final ParseField PARSE_NAME = new ParseField(NAME); private static final ParseField PARSE_INDEX_VERSION = new ParseField(INDEX_VERSION, "index-version"); @@ -498,20 +498,17 @@ public long totalSize() { */ @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(NAME, snapshot) - .field(INDEX_VERSION, indexVersion) - .field(START_TIME, startTime) - .field(TIME, time) - .field(INCREMENTAL_FILE_COUNT, incrementalFileCount) - .field(INCREMENTAL_SIZE, incrementalSize) - .startArray(FILES); + builder.field(NAME, snapshot); + builder.field(INDEX_VERSION, indexVersion); + builder.field(START_TIME, startTime); + builder.field(TIME, time); + builder.field(INCREMENTAL_FILE_COUNT, incrementalFileCount); + builder.field(INCREMENTAL_SIZE, incrementalSize); + builder.startArray(FILES); for (FileInfo fileInfo : indexFiles) { FileInfo.toXContent(fileInfo, builder, params); } - builder.endArray() - // BWC part - .field(NUMBER_OF_FILES, incrementalFileCount) - .field(TOTAL_SIZE, incrementalSize); + builder.endArray(); return builder; } @@ -553,6 +550,8 @@ public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) th incrementalFileCount = parser.intValue(); } else if (PARSE_INCREMENTAL_SIZE.match(currentFieldName, parser.getDeprecationHandler())) { incrementalSize = parser.longValue(); + } else { + throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); } } else if (token == XContentParser.Token.START_ARRAY) { if (PARSE_FILES.match(currentFieldName, parser.getDeprecationHandler())) { From 4a0bbc7d7ce1f687755db5a79cbe1a201ca2c097 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 11 May 2018 18:18:05 +0200 Subject: [PATCH 08/19] Include size of snapshot in snapshot metadata #29602 renamed nested objects to "incremental", "processed" and total" + moved timings out of "incremental" --- .../snapshots/status/SnapshotStats.java | 57 +++++++++++-------- .../snapshots/status/SnapshotStatusTests.java | 48 ++++++---------- 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index 8facba44692b4..2c0c16e336484 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -161,53 +161,60 @@ public void readFrom(StreamInput in) throws IOException { static final class Fields { static final String STATS = "stats"; - static final String INCREMENTAL_FILES = "incremental_files"; + static final String INCREMENTAL = "incremental"; + static final String PROCESSED = "processed"; + static final String TOTAL = "total"; + static final String COUNT = "count"; static final String SIZE = "size"; static final String SIZE_IN_BYTES = "size_in_bytes"; - static final String PROCESSED = "processed"; - static final String PROCESSED_SIZE = "processed_size"; - static final String PROCESSED_SIZE_IN_BYTES = "processed_size_in_bytes"; + static final String START_TIME_IN_MILLIS = "start_time_in_millis"; static final String TIME_IN_MILLIS = "time_in_millis"; static final String TIME = "time"; - static final String ALL_FILES = "all_files"; - // BWC static final String NUMBER_OF_FILES = "number_of_files"; static final String PROCESSED_FILES = "processed_files"; static final String TOTAL_SIZE = "total_size"; static final String TOTAL_SIZE_IN_BYTES = "total_size_in_bytes"; + static final String PROCESSED_SIZE_IN_BYTES = "processed_size_in_bytes"; + static final String PROCESSED_SIZE = "processed_size"; } @Override public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { - return builder.startObject(Fields.STATS) - // incremental_files starts - .startObject(Fields.INCREMENTAL_FILES) + builder.startObject(Fields.STATS) + // incremental starts + .startObject(Fields.INCREMENTAL) .field(Fields.COUNT, getIncrementalFileCount()) - .humanReadableField(Fields.SIZE_IN_BYTES,Fields.SIZE, new ByteSizeValue(getIncrementalSize())) - .field(Fields.PROCESSED, getProcessedFileCount()) - .humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES,Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())) - .field(Fields.START_TIME_IN_MILLIS, getStartTime()) - .humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())) - // incremental_files ends - .endObject() - // all_files starts - .startObject(Fields.ALL_FILES) + .humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(getIncrementalSize())) + // incremental ends + .endObject(); + + if (getProcessedFileCount() != getIncrementalFileCount()) { + // processed starts + builder.startObject(Fields.PROCESSED) + .field(Fields.COUNT, getProcessedFileCount()) + .humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(getProcessedSize())) + // processed ends + .endObject(); + } + // total starts + return builder.startObject(Fields.TOTAL) .field(Fields.COUNT, getTotalFileCount()) - .humanReadableField(Fields.SIZE_IN_BYTES,Fields.SIZE, new ByteSizeValue(getTotalSize())) + .humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(getTotalSize())) // all_files ends .endObject() + // timings stats + .field(Fields.START_TIME_IN_MILLIS, getStartTime()) + .humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())) // BWC part - .field(Fields.NUMBER_OF_FILES, getIncrementalFileCount()) - .field(Fields.PROCESSED_FILES, getProcessedFileCount()) - .humanReadableField(Fields.TOTAL_SIZE_IN_BYTES,Fields.TOTAL_SIZE, new ByteSizeValue(getIncrementalSize())) - .humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES,Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())) - .field(Fields.START_TIME_IN_MILLIS, getStartTime()) - .humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())) + .field(Fields.NUMBER_OF_FILES, getIncrementalFileCount()) + .field(Fields.PROCESSED_FILES, getProcessedFileCount()) + .humanReadableField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, new ByteSizeValue(getIncrementalSize())) + .humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES, Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())) // stats ends .endObject(); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java index a795c740850a6..cd0e0ff02e58b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java @@ -91,24 +91,20 @@ public void testToString() throws Exception { " \"total\" : " + totalShards + "\n" + " },\n" + " \"stats\" : {\n" + - " \"incremental_files\" : {\n" + + " \"incremental\" : {\n" + " \"count\" : 0,\n" + - " \"size_in_bytes\" : 0,\n" + - " \"processed\" : 0,\n" + - " \"processed_size_in_bytes\" : 0,\n" + - " \"start_time_in_millis\" : 0,\n" + - " \"time_in_millis\" : 0\n" + + " \"size_in_bytes\" : 0\n" + " },\n" + - " \"all_files\" : {\n" + + " \"total\" : {\n" + " \"count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + + " \"start_time_in_millis\" : 0,\n" + + " \"time_in_millis\" : 0,\n" + " \"number_of_files\" : 0,\n" + " \"processed_files\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + - " \"processed_size_in_bytes\" : 0,\n" + - " \"start_time_in_millis\" : 0,\n" + - " \"time_in_millis\" : 0\n" + + " \"processed_size_in_bytes\" : 0\n" + " },\n" + " \"indices\" : {\n" + " \"" + indexName + "\" : {\n" + @@ -121,47 +117,39 @@ public void testToString() throws Exception { " \"total\" : " + totalShards + "\n" + " },\n" + " \"stats\" : {\n" + - " \"incremental_files\" : {\n" + + " \"incremental\" : {\n" + " \"count\" : 0,\n" + - " \"size_in_bytes\" : 0,\n" + - " \"processed\" : 0,\n" + - " \"processed_size_in_bytes\" : 0,\n" + - " \"start_time_in_millis\" : 0,\n" + - " \"time_in_millis\" : 0\n" + + " \"size_in_bytes\" : 0\n" + " },\n" + - " \"all_files\" : {\n" + + " \"total\" : {\n" + " \"count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + + " \"start_time_in_millis\" : 0,\n" + + " \"time_in_millis\" : 0,\n" + " \"number_of_files\" : 0,\n" + " \"processed_files\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + - " \"processed_size_in_bytes\" : 0,\n" + - " \"start_time_in_millis\" : 0,\n" + - " \"time_in_millis\" : 0\n" + + " \"processed_size_in_bytes\" : 0\n" + " },\n" + " \"shards\" : {\n" + " \"" + shardId + "\" : {\n" + " \"stage\" : \"" + shardStage.toString() + "\",\n" + " \"stats\" : {\n" + - " \"incremental_files\" : {\n" + + " \"incremental\" : {\n" + " \"count\" : 0,\n" + - " \"size_in_bytes\" : 0,\n" + - " \"processed\" : 0,\n" + - " \"processed_size_in_bytes\" : 0,\n" + - " \"start_time_in_millis\" : 0,\n" + - " \"time_in_millis\" : 0\n" + + " \"size_in_bytes\" : 0\n" + " },\n" + - " \"all_files\" : {\n" + + " \"total\" : {\n" + " \"count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + + " \"start_time_in_millis\" : 0,\n" + + " \"time_in_millis\" : 0,\n" + " \"number_of_files\" : 0,\n" + " \"processed_files\" : 0,\n" + " \"total_size_in_bytes\" : 0,\n" + - " \"processed_size_in_bytes\" : 0,\n" + - " \"start_time_in_millis\" : 0,\n" + - " \"time_in_millis\" : 0\n" + + " \"processed_size_in_bytes\" : 0\n" + " }\n" + " }\n" + " }\n" + From 3ebc7693ad424429ca26c5cad68b907b7a35aa2e Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Mon, 14 May 2018 09:02:25 +0200 Subject: [PATCH 09/19] #29602 added snapshot stats section to docs --- docs/reference/modules/snapshots.asciidoc | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index ea3f99debb94e..4464972b75400 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -549,6 +549,64 @@ GET /_snapshot/my_backup/snapshot_1/_status // CONSOLE // TEST[continued] +Returns this: + +[source,js] +-------------------------------------------------- +{ + "snapshots": [ + { + "snapshot": "snapshot_1", + "repository": "my_backup", + "uuid": "XuBo4l4ISYiVg0nYUen9zg", + "state": "SUCCESS", + "include_global_state": true, + "shards_stats": { + "initializing": 0, + "started": 0, + "finalizing": 0, + "done": 5, + "failed": 0, + "total": 5 + }, + "stats": { + "incremental": { + "count": 8, + "size_in_bytes": 4704 + }, + "processed": { + "count": 7, + "size_in_bytes": 4254 + }, + "total": { + "count": 8, + "size_in_bytes": 4704 + }, + "start_time_in_millis": 1526280280355, + "time_in_millis": 358, + + + + "number_of_files": 8, + "processed_files": 8, + "total_size_in_bytes": 4704, + "processed_size_in_bytes": 4704 + }, + "indices": { + // ... + } + } + ] +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +Where `stats` provides snapshot details - how many files the snapshot consists of and sizes in bytes. +`processed` is an optional property that could be visible while snapshot has not been finished yet. +Properties go after `backward-compatible section` describe stats in compatible with version +`5.x`, `6.x` way - and will be dropped in next major version. + Multiple ids are also supported: [source,sh] From 828bb12da7d258c4322ff3573114e91a5742e41e Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 15 May 2018 13:10:44 +0200 Subject: [PATCH 10/19] #18543 use "file_count" to eliminate "incremental", "processed" and "total" naming confusion --- docs/reference/modules/snapshots.asciidoc | 7 +++---- .../cluster/snapshots/status/SnapshotStats.java | 8 ++++---- .../snapshots/status/SnapshotStatusTests.java | 12 ++++++------ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 4464972b75400..8c8cd4b80919b 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -571,15 +571,15 @@ Returns this: }, "stats": { "incremental": { - "count": 8, + "file_count": 8, "size_in_bytes": 4704 }, "processed": { - "count": 7, + "file_count": 7, "size_in_bytes": 4254 }, "total": { - "count": 8, + "file_count": 8, "size_in_bytes": 4704 }, "start_time_in_millis": 1526280280355, @@ -599,7 +599,6 @@ Returns this: ] } -------------------------------------------------- -// CONSOLE // TEST[continued] Where `stats` provides snapshot details - how many files the snapshot consists of and sizes in bytes. diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index 2c0c16e336484..c142b2650b7db 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -165,7 +165,7 @@ static final class Fields { static final String PROCESSED = "processed"; static final String TOTAL = "total"; - static final String COUNT = "count"; + static final String FILE_COUNT = "file_count"; static final String SIZE = "size"; static final String SIZE_IN_BYTES = "size_in_bytes"; @@ -188,7 +188,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.startObject(Fields.STATS) // incremental starts .startObject(Fields.INCREMENTAL) - .field(Fields.COUNT, getIncrementalFileCount()) + .field(Fields.FILE_COUNT, getIncrementalFileCount()) .humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(getIncrementalSize())) // incremental ends .endObject(); @@ -196,14 +196,14 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par if (getProcessedFileCount() != getIncrementalFileCount()) { // processed starts builder.startObject(Fields.PROCESSED) - .field(Fields.COUNT, getProcessedFileCount()) + .field(Fields.FILE_COUNT, getProcessedFileCount()) .humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(getProcessedSize())) // processed ends .endObject(); } // total starts return builder.startObject(Fields.TOTAL) - .field(Fields.COUNT, getTotalFileCount()) + .field(Fields.FILE_COUNT, getTotalFileCount()) .humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(getTotalSize())) // all_files ends .endObject() diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java index cd0e0ff02e58b..038fb063465f2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatusTests.java @@ -92,11 +92,11 @@ public void testToString() throws Exception { " },\n" + " \"stats\" : {\n" + " \"incremental\" : {\n" + - " \"count\" : 0,\n" + + " \"file_count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + " \"total\" : {\n" + - " \"count\" : 0,\n" + + " \"file_count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + " \"start_time_in_millis\" : 0,\n" + @@ -118,11 +118,11 @@ public void testToString() throws Exception { " },\n" + " \"stats\" : {\n" + " \"incremental\" : {\n" + - " \"count\" : 0,\n" + + " \"file_count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + " \"total\" : {\n" + - " \"count\" : 0,\n" + + " \"file_count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + " \"start_time_in_millis\" : 0,\n" + @@ -137,11 +137,11 @@ public void testToString() throws Exception { " \"stage\" : \"" + shardStage.toString() + "\",\n" + " \"stats\" : {\n" + " \"incremental\" : {\n" + - " \"count\" : 0,\n" + + " \"file_count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + " \"total\" : {\n" + - " \"count\" : 0,\n" + + " \"file_count\" : 0,\n" + " \"size_in_bytes\" : 0\n" + " },\n" + " \"start_time_in_millis\" : 0,\n" + From f3c2306b5fb1b2d60dc368df2e05cb15828ab0fa Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 22 May 2018 17:04:49 +0200 Subject: [PATCH 11/19] fix doc generation --- docs/reference/modules/snapshots.asciidoc | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 3a361b9cd16fd..3c6d6342522c7 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -599,26 +599,22 @@ Returns this: "start_time_in_millis": 1526280280355, "time_in_millis": 358, - - "number_of_files": 8, "processed_files": 8, "total_size_in_bytes": 4704, "processed_size_in_bytes": 4704 - }, - "indices": { - // ... } } ] } -------------------------------------------------- -// TEST[continued] +// TESTRESPONSE Where `stats` provides snapshot details - how many files the snapshot consists of and sizes in bytes. `processed` is an optional property that could be visible while snapshot has not been finished yet. -Properties go after `backward-compatible section` describe stats in compatible with version -`5.x`, `6.x` way - and will be dropped in next major version. + +*Note*: Properties `number_of_files`, `processed_files`, `total_size_in_bytes` and `processed_size_in_bytes` describe +stats in backward compatible with version `5.x`, `6.x` way - and will be dropped in next major version. Multiple ids are also supported: From 39e0190f5b9546e742105f33bec3c166ec15aa42 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 22 May 2018 19:51:03 +0200 Subject: [PATCH 12/19] adjusted docs, some comments --- docs/reference/modules/snapshots.asciidoc | 13 +++++++----- .../snapshots/status/SnapshotStats.java | 21 ++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 3c6d6342522c7..d443a4f2d4b4b 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -563,7 +563,7 @@ GET /_snapshot/my_backup/snapshot_1/_status // CONSOLE // TEST[continued] -Returns this: +The output looks similar to the following: [source,js] -------------------------------------------------- @@ -610,11 +610,14 @@ Returns this: -------------------------------------------------- // TESTRESPONSE -Where `stats` provides snapshot details - how many files the snapshot consists of and sizes in bytes. -`processed` is an optional property that could be visible while snapshot has not been finished yet. +The output is composed of different sections. The `stats` sub-object provides details on the number and size of files that were +snapshotted. As snapshots are incremental, copying only the Lucene segments that are not already in the repository, +the `stats` object contains a `total` section for all the files that are referenced by the snapshot, as well as an `incremental` section +for those files that actually needed to be copied over as part of the incremental snapshotting. In case of a snapshot that's still +n progress, there's also a `processed` section that contains information about the files that are in the process of being copied. -*Note*: Properties `number_of_files`, `processed_files`, `total_size_in_bytes` and `processed_size_in_bytes` describe -stats in backward compatible with version `5.x`, `6.x` way - and will be dropped in next major version. +_Note_: Properties `number_of_files`, `processed_files`, `total_size_in_bytes` and `processed_size_in_bytes` are used for +backward compatibility reasons with older 5.x and 6.x versions. These fields will be removed in Elasticsearch v7.0.0. Multiple ids are also supported: diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java index c142b2650b7db..f69f4ae552d81 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStats.java @@ -133,7 +133,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(processedSize); if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { - out.writeVInt(totalFileCount ); + out.writeVInt(totalFileCount); out.writeVLong(totalSize); } } @@ -202,20 +202,21 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par .endObject(); } // total starts - return builder.startObject(Fields.TOTAL) + builder.startObject(Fields.TOTAL) .field(Fields.FILE_COUNT, getTotalFileCount()) .humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(getTotalSize())) - // all_files ends - .endObject() - // timings stats - .field(Fields.START_TIME_IN_MILLIS, getStartTime()) - .humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())) - // BWC part - .field(Fields.NUMBER_OF_FILES, getIncrementalFileCount()) + // total ends + .endObject(); + // timings stats + builder.field(Fields.START_TIME_IN_MILLIS, getStartTime()) + .humanReadableField(Fields.TIME_IN_MILLIS, Fields.TIME, new TimeValue(getTime())); + + // BWC part + return builder.field(Fields.NUMBER_OF_FILES, getIncrementalFileCount()) .field(Fields.PROCESSED_FILES, getProcessedFileCount()) .humanReadableField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, new ByteSizeValue(getIncrementalSize())) .humanReadableField(Fields.PROCESSED_SIZE_IN_BYTES, Fields.PROCESSED_SIZE, new ByteSizeValue(getProcessedSize())) - // stats ends + // BWC part ends .endObject(); } From 34243af715d0b4629c1b011c3d6b8448dd835c81 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 23 May 2018 09:39:00 +0200 Subject: [PATCH 13/19] added section for 7.0 migration --- docs/reference/migration/migrate_7_0.asciidoc | 3 ++- .../migration/migrate_7_0/snapshotstats.asciidoc | 13 +++++++++++++ docs/reference/modules/snapshots.asciidoc | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 docs/reference/migration/migrate_7_0/snapshotstats.asciidoc diff --git a/docs/reference/migration/migrate_7_0.asciidoc b/docs/reference/migration/migrate_7_0.asciidoc index aea6d14fac9c8..815a4b12ac72c 100644 --- a/docs/reference/migration/migrate_7_0.asciidoc +++ b/docs/reference/migration/migrate_7_0.asciidoc @@ -34,7 +34,7 @@ Elasticsearch 6.x in order to be readable by Elasticsearch 7.x. * <> * <> * <> - +* <> include::migrate_7_0/aggregations.asciidoc[] include::migrate_7_0/analysis.asciidoc[] @@ -47,3 +47,4 @@ include::migrate_7_0/plugins.asciidoc[] include::migrate_7_0/api.asciidoc[] include::migrate_7_0/java.asciidoc[] include::migrate_7_0/settings.asciidoc[] +include::migrate_7_0/snapshotstats.asciidoc[] diff --git a/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc b/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc new file mode 100644 index 0000000000000..778aebbc4a12e --- /dev/null +++ b/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc @@ -0,0 +1,13 @@ +[[breaking_70_snapshotstats_changes]] +=== Snapshot stats changes + +Snapshot stats details are provided in a new structured way: + +* `total` section for all the files that are referenced by the snapshot +* `incremental` section for those files that actually needed to be copied over as part of the incremental snapshotting. +* In case of a snapshot that's still in progress, there's also a`processed` section for files that are in the process of being copied. + +==== Deprecated `number_of_files`, `processed_files`, `total_size_in_bytes` and `processed_size_in_bytes` snapshot stats properties have been removed + +* Properties `number_of_files` and `total_size_in_bytes` are removed and should be replaced by values of nested object `total`. +* Properties `processed_files` and `processed_size_in_bytes` are removed and should be replaced by values of nested object `processed`. \ No newline at end of file diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index d443a4f2d4b4b..1a017f4c4f20f 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -614,7 +614,7 @@ The output is composed of different sections. The `stats` sub-object provides de snapshotted. As snapshots are incremental, copying only the Lucene segments that are not already in the repository, the `stats` object contains a `total` section for all the files that are referenced by the snapshot, as well as an `incremental` section for those files that actually needed to be copied over as part of the incremental snapshotting. In case of a snapshot that's still -n progress, there's also a `processed` section that contains information about the files that are in the process of being copied. +in progress, there's also a `processed` section that contains information about the files that are in the process of being copied. _Note_: Properties `number_of_files`, `processed_files`, `total_size_in_bytes` and `processed_size_in_bytes` are used for backward compatibility reasons with older 5.x and 6.x versions. These fields will be removed in Elasticsearch v7.0.0. From 76fe2ac451a4e68b657ed21a9a65187a07e95a4a Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 23 May 2018 15:20:32 +0200 Subject: [PATCH 14/19] typos fixed --- docs/reference/migration/migrate_7_0/snapshotstats.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc b/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc index 778aebbc4a12e..6dbd24b13a1eb 100644 --- a/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc +++ b/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc @@ -3,9 +3,9 @@ Snapshot stats details are provided in a new structured way: -* `total` section for all the files that are referenced by the snapshot +* `total` section for all the files that are referenced by the snapshot. * `incremental` section for those files that actually needed to be copied over as part of the incremental snapshotting. -* In case of a snapshot that's still in progress, there's also a`processed` section for files that are in the process of being copied. +* In case of a snapshot that's still in progress, there's also a `processed` section for files that are in the process of being copied. ==== Deprecated `number_of_files`, `processed_files`, `total_size_in_bytes` and `processed_size_in_bytes` snapshot stats properties have been removed From c05b191e6ad7d92ecf8a3ba08be0e1cdc30b6b89 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 25 May 2018 12:36:15 +0200 Subject: [PATCH 15/19] added REST API test for snapshot/status (suspended before backporting is done) --- .../rest-api-spec/test/snapshot.status/10_basic.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index 838c12649748e..fccf644dbdaac 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -11,7 +11,9 @@ setup: --- "Get snapshot status": - + - skip: + version: " - 6.99.99" + reason: "backporting in progress: https://github.com/elastic/elasticsearch/pull/29602" - do: indices.create: index: test_index @@ -32,6 +34,14 @@ setup: snapshot: test_snapshot - is_true: snapshots + - match: { snapshots.0.snapshot: test_snapshot } + - match: { snapshots.0.state: SUCCESS } + - gt: { snapshots.0.stats.incremental.file_count: 0 } + - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } + - gt: { snapshots.0.stats.total.file_count: 0 } + - match: { snapshots.0.stats.start_time_in_millis: \d+ } + - gte: { snapshots.0.stats.time_in_millis: 0 } + --- "Get missing snapshot status throws an exception": From e1400a858b979b269882ab180c109bef00c9ee20 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 25 May 2018 13:46:04 +0200 Subject: [PATCH 16/19] added REST API test for snapshot/status with BWC fields --- .../test/snapshot.status/10_basic.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index fccf644dbdaac..cad22f0e946cf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -42,6 +42,34 @@ setup: - match: { snapshots.0.stats.start_time_in_millis: \d+ } - gte: { snapshots.0.stats.time_in_millis: 0 } +--- +"Get snapshot status with BWC fields": + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + snapshot.create: + repository: test_repo_status_1 + snapshot: test_snapshot + wait_for_completion: true + + - do: + snapshot.status: + repository: test_repo_status_1 + snapshot: test_snapshot + + - is_true: snapshots + - match: { snapshots.0.snapshot: test_snapshot } + - match: { snapshots.0.state: SUCCESS } + - gt: { snapshots.0.stats.number_of_files: 0 } + - gt: { snapshots.0.stats.processed_files: 0 } + - gt: { snapshots.0.stats.total_size_in_bytes: 0 } + - gt: { snapshots.0.stats.processed_size_in_bytes: 0 } --- "Get missing snapshot status throws an exception": From ccf7b22c719cdf362da082ad1296659dc8080b14 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 25 May 2018 15:45:36 +0200 Subject: [PATCH 17/19] fixed matching of time_in_millis/start_time_in_millis --- .../resources/rest-api-spec/test/snapshot.status/10_basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index cad22f0e946cf..66b4085a3effd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -39,8 +39,8 @@ setup: - gt: { snapshots.0.stats.incremental.file_count: 0 } - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 } - gt: { snapshots.0.stats.total.file_count: 0 } - - match: { snapshots.0.stats.start_time_in_millis: \d+ } - - gte: { snapshots.0.stats.time_in_millis: 0 } + - is_true: snapshots.0.stats.start_time_in_millis + - is_true: snapshots.0.stats.time_in_millis --- "Get snapshot status with BWC fields": From f2a33d6b62aec97213aca3f94433dd691149b98c Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 25 May 2018 17:29:05 +0200 Subject: [PATCH 18/19] fix snapshot name for bwc test --- .../resources/rest-api-spec/test/snapshot.status/10_basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index 66b4085a3effd..bcdb33cf99b31 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -55,7 +55,7 @@ setup: - do: snapshot.create: repository: test_repo_status_1 - snapshot: test_snapshot + snapshot: test_snapshot_bwc wait_for_completion: true - do: @@ -64,7 +64,7 @@ setup: snapshot: test_snapshot - is_true: snapshots - - match: { snapshots.0.snapshot: test_snapshot } + - match: { snapshots.0.snapshot: test_snapshot_bwc } - match: { snapshots.0.state: SUCCESS } - gt: { snapshots.0.stats.number_of_files: 0 } - gt: { snapshots.0.stats.processed_files: 0 } From 7df83efeac304a4aa3f6ad493e38669f1208713f Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Fri, 25 May 2018 18:51:12 +0200 Subject: [PATCH 19/19] fix snapshot name for bwc test --- .../resources/rest-api-spec/test/snapshot.status/10_basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml index bcdb33cf99b31..7e9d30830647b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml @@ -61,7 +61,7 @@ setup: - do: snapshot.status: repository: test_repo_status_1 - snapshot: test_snapshot + snapshot: test_snapshot_bwc - is_true: snapshots - match: { snapshots.0.snapshot: test_snapshot_bwc }