From 4650439ba7025058b9d16ef1bcecb44ceff7f787 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 12 Mar 2020 17:15:20 +0800 Subject: [PATCH 1/5] _cat/shards support path stats --- .../rest/action/cat/RestShardsAction.java | 9 +- .../action/cat/RestShardsActionTests.java | 146 ++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java index 40971a425e07e..fd83b2d34bea9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java @@ -200,6 +200,9 @@ protected Table getTableWithHeader(final RestRequest request) { table.addCell("warmer.total", "alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops"); table.addCell("warmer.total_time", "alias:wtt,warmerTotalTime;default:false;text-align:right;desc:time spent in warmers"); + table.addCell("path.data", "alias:pd,dataPath;default:false;text-align:right;desc:shard data path"); + table.addCell("path.state", "alias:ps,statsPath;default:false;text-align:right;desc:shard state path"); + table.endHeaders(); return table; } @@ -214,7 +217,8 @@ private static Object getOrNull(S stats, Function accessor, Functio return null; } - private Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) { + // package private for testing + Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsResponse stats) { Table table = getTableWithHeader(request); for (ShardRouting shard : state.getState().routingTable().allShards()) { @@ -351,6 +355,9 @@ private Table buildTable(RestRequest request, ClusterStateResponse state, Indice table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::total)); table.addCell(getOrNull(commonStats, CommonStats::getWarmer, WarmerStats::totalTime)); + table.addCell(getOrNull(shardStats, ShardStats::getDataPath, s -> s)); + table.addCell(getOrNull(shardStats, ShardStats::getStatePath, s -> s)); + table.endRow(); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java new file mode 100644 index 0000000000000..1a7966948ffd5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java @@ -0,0 +1,146 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.cat; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.common.Table; +import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.usage.UsageService; + +public class RestShardsActionTests extends ESTestCase { + + public void testBuildTable() { + final int numShards = randomIntBetween(1, 5); + DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + + List shardRoutings = new ArrayList<>(numShards); + Map shardStatsMap = new HashMap<>(); + String index = "index"; + for (int i = 0; i < numShards; i++) { + ShardRouting shardRouting = TestShardRouting.newShardRouting( + index, + i, + localNode.getId(), + randomBoolean(), + ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)) + ); + Path path = createTempDir().resolve("indices") + .resolve(shardRouting.shardId().getIndex().getUUID()) + .resolve(String.valueOf(shardRouting.shardId().id())); + ShardStats shardStats = new ShardStats( + shardRouting, + new ShardPath(false, path, path, shardRouting.shardId()), + null, + null, + null, + null + ); + shardStatsMap.put(shardRouting, shardStats); + shardRoutings.add(shardRouting); + } + + IndexStats indexStats = mock(IndexStats.class); + when(indexStats.getPrimaries()).thenReturn(new CommonStats()); + when(indexStats.getTotal()).thenReturn(new CommonStats()); + + IndicesStatsResponse stats = mock(IndicesStatsResponse.class); + when(stats.asMap()).thenReturn(shardStatsMap); + + DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); + when(discoveryNodes.get(localNode.getId())).thenReturn(localNode); + + ClusterStateResponse state = mock(ClusterStateResponse.class); + RoutingTable routingTable = mock(RoutingTable.class); + when(routingTable.allShards()).thenReturn(shardRoutings); + ClusterState clusterState = mock(ClusterState.class); + when(clusterState.routingTable()).thenReturn(routingTable); + when(clusterState.nodes()).thenReturn(discoveryNodes); + when(state.getState()).thenReturn(clusterState); + + final RestController restController = new RestController( + Collections.emptySet(), + null, + null, + null, + new UsageService(), + randomBoolean() + ); + final RestShardsAction action = new RestShardsAction(); + restController.registerHandler(action); + + final Table table = action.buildTable(new FakeRestRequest(), state, stats); + + // now, verify the table is correct + List headers = table.getHeaders(); + assertThat(headers.get(0).value, equalTo("index")); + assertThat(headers.get(1).value, equalTo("shard")); + assertThat(headers.get(2).value, equalTo("prirep")); + assertThat(headers.get(3).value, equalTo("state")); + assertThat(headers.get(4).value, equalTo("docs")); + assertThat(headers.get(5).value, equalTo("store")); + assertThat(headers.get(6).value, equalTo("ip")); + assertThat(headers.get(7).value, equalTo("id")); + assertThat(headers.get(8).value, equalTo("node")); + + final List> rows = table.getRows(); + assertThat(rows.size(), equalTo(numShards)); + + int i = 0; + for (final List row : rows) { + ShardRouting shardRouting = shardRoutings.get(i); + ShardStats shardStats = shardStatsMap.get(shardRouting); + assertThat(row.get(0).value, equalTo(shardRouting.getIndexName())); + assertThat(row.get(1).value, equalTo(shardRouting.getId())); + assertThat(row.get(2).value, equalTo(shardRouting.primary() ? "p" : "r")); + assertThat(row.get(3).value, equalTo(shardRouting.state())); + assertThat(row.get(6).value, equalTo(localNode.getHostAddress())); + assertThat(row.get(7).value, equalTo(localNode.getId())); + assertThat(row.get(69).value, equalTo(shardStats.getDataPath())); + assertThat(row.get(70).value, equalTo(shardStats.getStatePath())); + i++; + } + } +} From 6ee62cca3730d4be2006ca02e63ddaa1d8613c9b Mon Sep 17 00:00:00 2001 From: weizijun Date: Sat, 14 Mar 2020 21:19:16 +0800 Subject: [PATCH 2/5] fix some style case --- .../action/cat/RestShardsActionTests.java | 61 ++++++------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java index 1a7966948ffd5..8c9cd33e7082d 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java @@ -19,17 +19,6 @@ package org.elasticsearch.rest.action.cat; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.stats.CommonStats; @@ -45,10 +34,19 @@ import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.Table; import org.elasticsearch.index.shard.ShardPath; -import org.elasticsearch.rest.RestController; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; -import org.elasticsearch.usage.UsageService; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class RestShardsActionTests extends ESTestCase { @@ -60,24 +58,12 @@ public void testBuildTable() { Map shardStatsMap = new HashMap<>(); String index = "index"; for (int i = 0; i < numShards; i++) { - ShardRouting shardRouting = TestShardRouting.newShardRouting( - index, - i, - localNode.getId(), - randomBoolean(), - ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)) - ); - Path path = createTempDir().resolve("indices") - .resolve(shardRouting.shardId().getIndex().getUUID()) + ShardRouting shardRouting = TestShardRouting.newShardRouting( index, i, localNode.getId(), randomBoolean(), + ShardRoutingState.fromValue((byte) randomIntBetween(2, 3))); + Path path = createTempDir().resolve("indices").resolve(shardRouting.shardId().getIndex().getUUID()) .resolve(String.valueOf(shardRouting.shardId().id())); - ShardStats shardStats = new ShardStats( - shardRouting, - new ShardPath(false, path, path, shardRouting.shardId()), - null, - null, - null, - null - ); + ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, shardRouting.shardId()), + null, null, null, null); shardStatsMap.put(shardRouting, shardStats); shardRoutings.add(shardRouting); } @@ -100,17 +86,7 @@ public void testBuildTable() { when(clusterState.nodes()).thenReturn(discoveryNodes); when(state.getState()).thenReturn(clusterState); - final RestController restController = new RestController( - Collections.emptySet(), - null, - null, - null, - new UsageService(), - randomBoolean() - ); final RestShardsAction action = new RestShardsAction(); - restController.registerHandler(action); - final Table table = action.buildTable(new FakeRestRequest(), state, stats); // now, verify the table is correct @@ -128,9 +104,9 @@ public void testBuildTable() { final List> rows = table.getRows(); assertThat(rows.size(), equalTo(numShards)); - int i = 0; + Iterator shardRoutingsIt = shardRoutings.iterator(); for (final List row : rows) { - ShardRouting shardRouting = shardRoutings.get(i); + ShardRouting shardRouting = shardRoutingsIt.next(); ShardStats shardStats = shardStatsMap.get(shardRouting); assertThat(row.get(0).value, equalTo(shardRouting.getIndexName())); assertThat(row.get(1).value, equalTo(shardRouting.getId())); @@ -140,7 +116,6 @@ public void testBuildTable() { assertThat(row.get(7).value, equalTo(localNode.getId())); assertThat(row.get(69).value, equalTo(shardStats.getDataPath())); assertThat(row.get(70).value, equalTo(shardStats.getStatePath())); - i++; } } } From de173dee67dffa8b2c2b270575b6705931e2ce1f Mon Sep 17 00:00:00 2001 From: weizijun Date: Sat, 14 Mar 2020 22:14:38 +0800 Subject: [PATCH 3/5] fix some style case --- .../elasticsearch/rest/action/cat/RestShardsActionTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java index 8c9cd33e7082d..f661521376cf5 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java @@ -58,8 +58,8 @@ public void testBuildTable() { Map shardStatsMap = new HashMap<>(); String index = "index"; for (int i = 0; i < numShards; i++) { - ShardRouting shardRouting = TestShardRouting.newShardRouting( index, i, localNode.getId(), randomBoolean(), - ShardRoutingState.fromValue((byte) randomIntBetween(2, 3))); + ShardRoutingState shardRoutingState = ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)); + ShardRouting shardRouting = TestShardRouting.newShardRouting(index, i, localNode.getId(), randomBoolean(), shardRoutingState); Path path = createTempDir().resolve("indices").resolve(shardRouting.shardId().getIndex().getUUID()) .resolve(String.valueOf(shardRouting.shardId().id())); ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, shardRouting.shardId()), From 43da1816c25ee73e1485906c113194cd343ea5bb Mon Sep 17 00:00:00 2001 From: weizijun Date: Sun, 15 Mar 2020 11:07:01 +0800 Subject: [PATCH 4/5] fix rest-api-spec cat.shards error --- .../resources/rest-api-spec/test/cat.shards/10_basic.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml index de5e632975752..f9cb2bcd959af 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -1,8 +1,8 @@ --- "Help": - skip: - version: " - 7.1.99" - reason: external refresh stats were added in 7.2.0 + version: " - 7.6.99" + reason: shard path stats were added in 7.7.0 - do: cat.shards: help: true @@ -78,6 +78,8 @@ warmer.current .+ \n warmer.total .+ \n warmer.total_time .+ \n + path.data .+ \n + path.state .+ \n $/ --- "Test cat shards output": From 9b11fd948996987245f2a2df5f1a2b024fd58e8a Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 17 Mar 2020 15:14:59 +0800 Subject: [PATCH 5/5] fix rest-api-spec cat.shards bwc error --- .../main/resources/rest-api-spec/test/cat.shards/10_basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml index f9cb2bcd959af..aa4abc7a11eae 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -1,8 +1,8 @@ --- "Help": - skip: - version: " - 7.6.99" - reason: shard path stats were added in 7.7.0 + version: " - 7.99.99" + reason: shard path stats were added in 8.0.0 - do: cat.shards: help: true