From 1cf868b91243c9049e6eb9f43a035ee4b3988559 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 16 Jul 2021 12:18:42 -0500 Subject: [PATCH 1/8] Adding shard count to node stats api Added a shards section to each node returned by the node stats api. Currently it only contains a count of shards --- docs/reference/cluster/nodes-stats.asciidoc | 12 ++++ .../rest-api-spec/api/nodes.stats.json | 6 +- .../test/nodes.stats/11_indices_metrics.yml | 12 +++- .../indices/stats/IndexStatsIT.java | 8 ++- .../admin/indices/stats/CommonStats.java | 28 ++++++++- .../admin/indices/stats/CommonStatsFlags.java | 3 +- .../index/shard/SimpleShardStats.java | 61 +++++++++++++++++++ .../index/shard/SimpleShardStatsTest.java | 45 ++++++++++++++ 8 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/shard/SimpleShardStats.java create mode 100644 server/src/test/java/org/elasticsearch/index/shard/SimpleShardStatsTest.java diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 0a764b6095ff1..c9d362a58185c 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -959,6 +959,18 @@ Time by which recovery operations were delayed due to throttling. Time in milliseconds recovery operations were delayed due to throttling. ======= + +`shards`:: +(object) +Contains statistics about all shards assigned to the node. ++ +.Properties of `shards` +[%collapsible%open] +======= +`total_count`:: +(integer) +The total number of shards assigned to the node. +====== ====== [[cluster-nodes-stats-api-response-body-os]] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json index acea11146d384..7a13a6c1033c5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json @@ -127,7 +127,8 @@ "segments", "store", "warmer", - "bulk" + "bulk", + "shards" ], "description":"Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified." } @@ -175,7 +176,8 @@ "segments", "store", "warmer", - "bulk" + "bulk", + "shards" ], "description":"Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified." }, diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index a3aa6e3593177..c604eff6a84dc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -25,6 +25,7 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.shards --- "Metric - _all": @@ -53,6 +54,7 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.shards --- "Metric - indices _all": @@ -81,6 +83,7 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.shards --- "Metric - one": @@ -109,6 +112,7 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_false: nodes.$node_id.indices.shards --- "Metric - multi": @@ -120,7 +124,7 @@ nodes._arbitrary_key_: node_id - do: - nodes.stats: { metric: indices, index_metric: [ store, get, merge ] } + nodes.stats: { metric: indices, index_metric: [ store, get, merge, shards ] } - is_false: nodes.$node_id.indices.docs - is_true: nodes.$node_id.indices.store @@ -137,6 +141,7 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.shards --- @@ -166,6 +171,7 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_false: nodes.$node_id.indices.shards --- "Metric - _all include_segment_file_sizes": @@ -194,6 +200,7 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.shards - is_true: nodes.$node_id.indices.segments.file_sizes --- @@ -223,6 +230,7 @@ - is_true: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_false: nodes.$node_id.indices.shards - is_true: nodes.$node_id.indices.segments.file_sizes --- @@ -254,6 +262,7 @@ - is_true: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery + - is_false: nodes.$node_id.indices.shards --- "Metric - _all include_unloaded_segments": @@ -284,6 +293,7 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery + - is_true: nodes.$node_id.indices.shards --- "Metric - http": diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java index a05cd5e6fd48e..fc2594752645d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -735,7 +735,7 @@ public void testEncodeDecodeCommonStats() throws IOException { public void testFlagOrdinalOrder() { Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh, Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Completion, Flag.Segments, - Flag.Translog, Flag.RequestCache, Flag.Recovery, Flag.Bulk}; + Flag.Translog, Flag.RequestCache, Flag.Recovery, Flag.Bulk, Flag.Shards}; assertThat(flags.length, equalTo(Flag.values().length)); for (int i = 0; i < flags.length; i++) { @@ -914,6 +914,10 @@ private static void set(Flag flag, IndicesStatsRequestBuilder builder, boolean s case Bulk: builder.setBulk(set); break; + case Shards: + // We don't actually expose shards in IndexStats, but this test fails if it isn't handled + builder.request().flags().set(Flag.Shards, set); + break; default: fail("new flag? " + flag); break; @@ -956,6 +960,8 @@ private static boolean isSet(Flag flag, CommonStats response) { return response.getRecoveryStats() != null; case Bulk: return response.getBulk() != null; + case Shards: + return response.getShards() != null; default: fail("new flag? " + flag); return false; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index 772cb100a534c..d9c8499294b76 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexingStats; +import org.elasticsearch.index.shard.SimpleShardStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.warmer.WarmerStats; @@ -96,6 +97,9 @@ public class CommonStats implements Writeable, ToXContentFragment { @Nullable public BulkStats bulk; + @Nullable + public SimpleShardStats shards; + public CommonStats() { this(CommonStatsFlags.NONE); } @@ -156,6 +160,9 @@ public CommonStats(CommonStatsFlags flags) { case Bulk: bulk = new BulkStats(); break; + case Shards: + shards = new SimpleShardStats(); + break; default: throw new IllegalStateException("Unknown Flag: " + flag); } @@ -218,6 +225,10 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C case Bulk: bulk = indexShard.bulkStats(); break; + case Shards: + // Setting to 1 because the single IndexShard passed to this method implies 1 shard + shards = new SimpleShardStats(1); + break; default: throw new IllegalStateException("Unknown Flag: " + flag); } @@ -246,6 +257,7 @@ public CommonStats(StreamInput in) throws IOException { recoveryStats = in.readOptionalWriteable(RecoveryStats::new); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { bulk = in.readOptionalWriteable(BulkStats::new); + shards = in.readOptionalWriteable(SimpleShardStats::new); } } @@ -269,6 +281,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(recoveryStats); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { out.writeOptionalWriteable(bulk); + out.writeOptionalWriteable(shards); } } @@ -410,6 +423,14 @@ public void add(CommonStats stats) { } else { bulk.add(stats.getBulk()); } + if (stats.shards != null) { + if (shards == null) { + shards = stats.shards; + } + else { + shards = shards.add(stats.shards); + } + } } @Nullable @@ -497,6 +518,11 @@ public BulkStats getBulk() { return bulk; } + @Nullable + public SimpleShardStats getShards() { + return shards; + } + /** * Utility method which computes total memory by adding * FieldData, PercolatorCache, Segments (memory, index writer, version map) @@ -522,7 +548,7 @@ public ByteSizeValue getTotalMemory() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { final Stream stream = Arrays.stream(new ToXContent[] { - docs, store, indexing, get, search, merge, refresh, flush, warmer, queryCache, + docs, shards, store, indexing, get, search, merge, refresh, flush, warmer, queryCache, fieldData, completion, segments, translog, requestCache, recoveryStats, bulk}) .filter(Objects::nonNull); for (ToXContent toXContent : ((Iterable)stream::iterator)) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index 70e677cb6e055..dd89fa3323d02 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -217,7 +217,8 @@ public enum Flag { // 14 was previously used for Suggest RequestCache("request_cache", 15), Recovery("recovery", 16), - Bulk("bulk", 17); + Bulk("bulk", 17), + Shards("shards", 18); private final String restName; private final int index; diff --git a/server/src/main/java/org/elasticsearch/index/shard/SimpleShardStats.java b/server/src/main/java/org/elasticsearch/index/shard/SimpleShardStats.java new file mode 100644 index 0000000000000..2e67a7e220510 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/shard/SimpleShardStats.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; + +public class SimpleShardStats implements Writeable, ToXContentFragment { + + private final long totalCount; + + public SimpleShardStats() { + totalCount = 0; + } + + public SimpleShardStats(StreamInput in) throws IOException { + totalCount = in.readVLong(); + } + + public SimpleShardStats(long totalCount) { + this.totalCount = totalCount; + } + + public long getTotalCount() { + return this.totalCount; + } + + public SimpleShardStats add(@Nullable SimpleShardStats other) { + return new SimpleShardStats(this.totalCount + (other == null ? 0 : other.totalCount)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.SHARDS); + builder.field(Fields.TOTAL_COUNT, totalCount); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(totalCount); + } + + static final class Fields { + static final String SHARDS = "shards"; + static final String TOTAL_COUNT = "total_count"; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/SimpleShardStatsTest.java b/server/src/test/java/org/elasticsearch/index/shard/SimpleShardStatsTest.java new file mode 100644 index 0000000000000..8571ccbfd4f02 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/SimpleShardStatsTest.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; +import org.junit.Assert; + +import static org.hamcrest.Matchers.equalTo; + +class SimpleShardStatsTest extends ESTestCase { + + public void testAdd() { + SimpleShardStats shardStats1 = new SimpleShardStats(5); + SimpleShardStats shardStats2 = new SimpleShardStats(8); + SimpleShardStats combinedShardStats = shardStats1.add(shardStats2); + Assert.assertEquals(13, combinedShardStats.getTotalCount()); + Assert.assertEquals(5, shardStats1.getTotalCount()); + Assert.assertEquals(8, shardStats2.getTotalCount()); + SimpleShardStats noCountGiven = new SimpleShardStats(); + Assert.assertEquals(0, noCountGiven.getTotalCount()); + Assert.assertEquals(8, shardStats2.add(null).getTotalCount()); + Assert.assertEquals(8, shardStats2.getTotalCount()); + } + + public void testSerialize() throws Exception { + SimpleShardStats originalStats = new SimpleShardStats(5); + try (BytesStreamOutput out = new BytesStreamOutput()) { + originalStats.writeTo(out); + BytesReference bytes = out.bytes(); + try (StreamInput in = bytes.streamInput()) { + SimpleShardStats cloneStats = new SimpleShardStats(in); + assertThat(cloneStats.getTotalCount(), equalTo(originalStats.getTotalCount())); + } + } + } +} From 92f6873cbfc358b1067099448ca9499cda71d2e9 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 16 Jul 2021 14:13:37 -0500 Subject: [PATCH 2/8] adding a yaml test --- .../test/nodes.stats/11_indices_metrics.yml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index c604eff6a84dc..5109240eb8bf0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -324,3 +324,35 @@ - gte: { nodes.$node_id.http.clients.0.request_size_bytes: 0 } # values for clients.0.closed_time_millis, clients.0.x_forwarded_for, and clients.0.x_opaque_id are often # null and cannot be tested here + +--- +"indices shards total count test": + + - skip: + features: ["allowed_warnings", arbitrary_key] + + - do: + indices.create: + index: index1 + body: + settings: + number_of_shards: "3" + number_of_replicas: "0" + + - do: + indices.create: + index: index2 + body: + settings: + number_of_shards: "2" + number_of_replicas: "0" + + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: { metric: _all } + + - match: { nodes.$node_id.indices.shards.total_count: 5 } From 04605f9ca66449c0c9e98453b123d5b8e8a7608c Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 19 Jul 2021 12:28:07 -0500 Subject: [PATCH 3/8] Attemtping to get IT tests to pass --- docs/reference/cluster/nodes-stats.asciidoc | 2 +- .../test/nodes.stats/11_indices_metrics.yml | 52 +++++++++++++------ 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index c9d362a58185c..033f202169985 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -970,7 +970,7 @@ Contains statistics about all shards assigned to the node. `total_count`:: (integer) The total number of shards assigned to the node. -====== +======= ====== [[cluster-nodes-stats-api-response-body-os]] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 5109240eb8bf0..1849f2c806981 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -25,7 +25,6 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery - - is_true: nodes.$node_id.indices.shards --- "Metric - _all": @@ -54,7 +53,6 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery - - is_true: nodes.$node_id.indices.shards --- "Metric - indices _all": @@ -83,7 +81,6 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery - - is_true: nodes.$node_id.indices.shards --- "Metric - one": @@ -124,7 +121,7 @@ nodes._arbitrary_key_: node_id - do: - nodes.stats: { metric: indices, index_metric: [ store, get, merge, shards ] } + nodes.stats: { metric: indices, index_metric: [ store, get, merge ] } - is_false: nodes.$node_id.indices.docs - is_true: nodes.$node_id.indices.store @@ -141,7 +138,6 @@ - is_false: nodes.$node_id.indices.segments - is_false: nodes.$node_id.indices.translog - is_false: nodes.$node_id.indices.recovery - - is_true: nodes.$node_id.indices.shards --- @@ -200,7 +196,6 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery - - is_true: nodes.$node_id.indices.shards - is_true: nodes.$node_id.indices.segments.file_sizes --- @@ -293,7 +288,6 @@ - is_true: nodes.$node_id.indices.segments - is_true: nodes.$node_id.indices.translog - is_true: nodes.$node_id.indices.recovery - - is_true: nodes.$node_id.indices.shards --- "Metric - http": @@ -325,26 +319,52 @@ # values for clients.0.closed_time_millis, clients.0.x_forwarded_for, and clients.0.x_opaque_id are often # null and cannot be tested here +--- +"Metric - blank for indices shards": + - skip: + features: [arbitrary_key] + version: " - 7.14.99" + reason: "total shard count added in version 7.15" + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: {} + + - is_true: nodes.$node_id.indices.shards + +--- +"Metric - _all for indices shards": + - skip: + features: [arbitrary_key] + version: " - 7.14.99" + reason: "total shard count added in version 7.15" + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: { metric: _all } + + - is_true: nodes.$node_id.indices.shards + --- "indices shards total count test": - skip: features: ["allowed_warnings", arbitrary_key] + version: " - 7.14.99" + reason: "total shard count added in version 7.15" - do: indices.create: index: index1 body: settings: - number_of_shards: "3" - number_of_replicas: "0" - - - do: - indices.create: - index: index2 - body: - settings: - number_of_shards: "2" + number_of_shards: "5" number_of_replicas: "0" - do: From 26dae7f9004b2009b482c3760bedff9cc49e62a8 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 19 Jul 2021 14:56:21 -0500 Subject: [PATCH 4/8] Fixing IT tests --- .../test/nodes.stats/11_indices_metrics.yml | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 1849f2c806981..0253247cbf391 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -323,8 +323,8 @@ "Metric - blank for indices shards": - skip: features: [arbitrary_key] - version: " - 7.14.99" - reason: "total shard count added in version 7.15" + version: " - 7.99.99" + reason: "total shard count added in version 8.0" - do: nodes.info: {} - set: @@ -334,13 +334,14 @@ nodes.stats: {} - is_true: nodes.$node_id.indices.shards + - match: { nodes.$node_id.indices.shards.total_count: 0 } --- "Metric - _all for indices shards": - skip: features: [arbitrary_key] - version: " - 7.14.99" - reason: "total shard count added in version 7.15" + version: " - 7.99.99" + reason: "total shard count added in version 8.0" - do: nodes.info: {} - set: @@ -350,14 +351,16 @@ nodes.stats: { metric: _all } - is_true: nodes.$node_id.indices.shards + - match: { nodes.$node_id.indices.shards.total_count: 0 } + --- "indices shards total count test": - skip: features: ["allowed_warnings", arbitrary_key] - version: " - 7.14.99" - reason: "total shard count added in version 7.15" + version: " - 7.99.99" + reason: "total shard count added in version 8.0" - do: indices.create: @@ -367,6 +370,14 @@ number_of_shards: "5" number_of_replicas: "0" + - do: + indices.create: + index: index2 + body: + settings: + number_of_shards: "3" + number_of_replicas: "1" + - do: nodes.info: {} - set: @@ -375,4 +386,4 @@ - do: nodes.stats: { metric: _all } - - match: { nodes.$node_id.indices.shards.total_count: 5 } + - gte: { nodes.$node_id.indices.shards.total_count: 1 } From 92b082e35d422c50e1bc1ea7f378b2f65bdf7200 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 26 Jul 2021 11:17:50 -0500 Subject: [PATCH 5/8] Renamed SimpleShardStats to ShardCountStats --- .../action/admin/indices/stats/CommonStats.java | 12 ++++++------ ...{SimpleShardStats.java => ShardCountStats.java} | 12 ++++++------ ...hardStatsTest.java => ShardCountStatsTest.java} | 14 +++++++------- 3 files changed, 19 insertions(+), 19 deletions(-) rename server/src/main/java/org/elasticsearch/index/shard/{SimpleShardStats.java => ShardCountStats.java} (80%) rename server/src/test/java/org/elasticsearch/index/shard/{SimpleShardStatsTest.java => ShardCountStatsTest.java} (76%) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index d9c8499294b76..6d7cb1c0c919a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -32,7 +32,7 @@ import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexingStats; -import org.elasticsearch.index.shard.SimpleShardStats; +import org.elasticsearch.index.shard.ShardCountStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.warmer.WarmerStats; @@ -98,7 +98,7 @@ public class CommonStats implements Writeable, ToXContentFragment { public BulkStats bulk; @Nullable - public SimpleShardStats shards; + public ShardCountStats shards; public CommonStats() { this(CommonStatsFlags.NONE); @@ -161,7 +161,7 @@ public CommonStats(CommonStatsFlags flags) { bulk = new BulkStats(); break; case Shards: - shards = new SimpleShardStats(); + shards = new ShardCountStats(); break; default: throw new IllegalStateException("Unknown Flag: " + flag); @@ -227,7 +227,7 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C break; case Shards: // Setting to 1 because the single IndexShard passed to this method implies 1 shard - shards = new SimpleShardStats(1); + shards = new ShardCountStats(1); break; default: throw new IllegalStateException("Unknown Flag: " + flag); @@ -257,7 +257,7 @@ public CommonStats(StreamInput in) throws IOException { recoveryStats = in.readOptionalWriteable(RecoveryStats::new); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { bulk = in.readOptionalWriteable(BulkStats::new); - shards = in.readOptionalWriteable(SimpleShardStats::new); + shards = in.readOptionalWriteable(ShardCountStats::new); } } @@ -519,7 +519,7 @@ public BulkStats getBulk() { } @Nullable - public SimpleShardStats getShards() { + public ShardCountStats getShards() { return shards; } diff --git a/server/src/main/java/org/elasticsearch/index/shard/SimpleShardStats.java b/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java similarity index 80% rename from server/src/main/java/org/elasticsearch/index/shard/SimpleShardStats.java rename to server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java index 2e67a7e220510..5a29da772f283 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/SimpleShardStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java @@ -17,19 +17,19 @@ import java.io.IOException; -public class SimpleShardStats implements Writeable, ToXContentFragment { +public class ShardCountStats implements Writeable, ToXContentFragment { private final long totalCount; - public SimpleShardStats() { + public ShardCountStats() { totalCount = 0; } - public SimpleShardStats(StreamInput in) throws IOException { + public ShardCountStats(StreamInput in) throws IOException { totalCount = in.readVLong(); } - public SimpleShardStats(long totalCount) { + public ShardCountStats(long totalCount) { this.totalCount = totalCount; } @@ -37,8 +37,8 @@ public long getTotalCount() { return this.totalCount; } - public SimpleShardStats add(@Nullable SimpleShardStats other) { - return new SimpleShardStats(this.totalCount + (other == null ? 0 : other.totalCount)); + public ShardCountStats add(@Nullable ShardCountStats other) { + return new ShardCountStats(this.totalCount + (other == null ? 0 : other.totalCount)); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/shard/SimpleShardStatsTest.java b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java similarity index 76% rename from server/src/test/java/org/elasticsearch/index/shard/SimpleShardStatsTest.java rename to server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java index 8571ccbfd4f02..bbe9796d1e65f 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/SimpleShardStatsTest.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java @@ -16,28 +16,28 @@ import static org.hamcrest.Matchers.equalTo; -class SimpleShardStatsTest extends ESTestCase { +class ShardCountStatsTest extends ESTestCase { public void testAdd() { - SimpleShardStats shardStats1 = new SimpleShardStats(5); - SimpleShardStats shardStats2 = new SimpleShardStats(8); - SimpleShardStats combinedShardStats = shardStats1.add(shardStats2); + ShardCountStats shardStats1 = new ShardCountStats(5); + ShardCountStats shardStats2 = new ShardCountStats(8); + ShardCountStats combinedShardStats = shardStats1.add(shardStats2); Assert.assertEquals(13, combinedShardStats.getTotalCount()); Assert.assertEquals(5, shardStats1.getTotalCount()); Assert.assertEquals(8, shardStats2.getTotalCount()); - SimpleShardStats noCountGiven = new SimpleShardStats(); + ShardCountStats noCountGiven = new ShardCountStats(); Assert.assertEquals(0, noCountGiven.getTotalCount()); Assert.assertEquals(8, shardStats2.add(null).getTotalCount()); Assert.assertEquals(8, shardStats2.getTotalCount()); } public void testSerialize() throws Exception { - SimpleShardStats originalStats = new SimpleShardStats(5); + ShardCountStats originalStats = new ShardCountStats(5); try (BytesStreamOutput out = new BytesStreamOutput()) { originalStats.writeTo(out); BytesReference bytes = out.bytes(); try (StreamInput in = bytes.streamInput()) { - SimpleShardStats cloneStats = new SimpleShardStats(in); + ShardCountStats cloneStats = new ShardCountStats(in); assertThat(cloneStats.getTotalCount(), equalTo(originalStats.getTotalCount())); } } From fbc83b4cc5ad891900a3952685895b80b413bff1 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 26 Jul 2021 11:30:30 -0500 Subject: [PATCH 6/8] improving unit test --- .../index/shard/ShardCountStats.java | 16 +++++++++++ .../index/shard/ShardCountStatsTest.java | 28 ++++++++----------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java b/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java index 5a29da772f283..cbbaff45c6961 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.shard; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -58,4 +59,19 @@ static final class Fields { static final String SHARDS = "shards"; static final String TOTAL_COUNT = "total_count"; } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + + @Override + public boolean equals(Object o) { + return (o instanceof ShardCountStats) && totalCount == ((ShardCountStats) o).totalCount; + } + + @Override + public int hashCode() { + return Long.hashCode(totalCount); + } } diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java index bbe9796d1e65f..4f36688c7729e 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java @@ -8,15 +8,11 @@ package org.elasticsearch.index.shard; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.junit.Assert; -import static org.hamcrest.Matchers.equalTo; - -class ShardCountStatsTest extends ESTestCase { +public class ShardCountStatsTest extends AbstractWireSerializingTestCase { public void testAdd() { ShardCountStats shardStats1 = new ShardCountStats(5); @@ -31,15 +27,13 @@ public void testAdd() { Assert.assertEquals(8, shardStats2.getTotalCount()); } - public void testSerialize() throws Exception { - ShardCountStats originalStats = new ShardCountStats(5); - try (BytesStreamOutput out = new BytesStreamOutput()) { - originalStats.writeTo(out); - BytesReference bytes = out.bytes(); - try (StreamInput in = bytes.streamInput()) { - ShardCountStats cloneStats = new ShardCountStats(in); - assertThat(cloneStats.getTotalCount(), equalTo(originalStats.getTotalCount())); - } - } + @Override + protected Writeable.Reader instanceReader() { + return ShardCountStats::new; + } + + @Override + protected ShardCountStats createTestInstance() { + return new ShardCountStats(randomIntBetween(0, Integer.MAX_VALUE)); } } From e59b047abe9ebb4c2bda6f60139ef8cce139e5d8 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 26 Jul 2021 13:29:02 -0500 Subject: [PATCH 7/8] Making a unit test non public --- .../java/org/elasticsearch/index/shard/ShardCountStatsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java index 4f36688c7729e..9e7d130182032 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardCountStatsTest.java @@ -12,7 +12,7 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.junit.Assert; -public class ShardCountStatsTest extends AbstractWireSerializingTestCase { +class ShardCountStatsTest extends AbstractWireSerializingTestCase { public void testAdd() { ShardCountStats shardStats1 = new ShardCountStats(5); From 374bc9268b594cb3d4091c2d682c4655ebda8c5b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 27 Jul 2021 09:09:33 -0500 Subject: [PATCH 8/8] modifying toString() --- .../java/org/elasticsearch/index/shard/ShardCountStats.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java b/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java index cbbaff45c6961..5ea65c35691ea 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardCountStats.java @@ -62,7 +62,7 @@ static final class Fields { @Override public String toString() { - return Strings.toString(this, true, true); + return Strings.toString(this); } @Override