diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java index 497ab938856b..29679e6fb6f4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java @@ -161,6 +161,12 @@ default double getAverageLoad() { */ Map getTableRegionStatesCount(); + /** + * Provide the list of master tasks + */ + @Nullable + List getMasterTasks(); + /** * Kinds of ClusterMetrics */ @@ -213,5 +219,9 @@ enum Option { * metrics about table to no of regions status count */ TABLE_TO_REGIONS_COUNT, + /** + * metrics about monitored tasks + */ + TASKS, } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java index 493fe71b8b0f..011f93f9fe90 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java @@ -83,6 +83,10 @@ public static ClusterStatusProtos.ClusterStatus toClusterStatus(ClusterMetrics m if (metrics.getMasterName() != null) { builder.setMaster(ProtobufUtil.toServerName((metrics.getMasterName()))); } + if (metrics.getMasterTasks() != null) { + builder.addAllMasterTasks(metrics.getMasterTasks().stream() + .map(t -> ProtobufUtil.toServerTask(t)).collect(Collectors.toList())); + } if (metrics.getBalancerOn() != null) { builder.setBalancerOn(metrics.getBalancerOn()); } @@ -122,7 +126,9 @@ public static ClusterMetrics toClusterMetrics( proto.getTableRegionStatesCountList().stream() .collect(Collectors.toMap( e -> ProtobufUtil.toTableName(e.getTableName()), - e -> ProtobufUtil.toTableRegionStatesCount(e.getRegionStatesCount())))); + e -> ProtobufUtil.toTableRegionStatesCount(e.getRegionStatesCount())))) + .setMasterTasks(proto.getMasterTasksList().stream() + .map(t -> ProtobufUtil.getServerTask(t)).collect(Collectors.toList())); if (proto.hasClusterId()) { builder.setClusterId(ClusterId.convert(proto.getClusterId()).toString()); } @@ -164,6 +170,7 @@ public static ClusterMetrics.Option toOption(ClusterStatusProtos.Option option) case SERVERS_NAME: return ClusterMetrics.Option.SERVERS_NAME; case MASTER_INFO_PORT: return ClusterMetrics.Option.MASTER_INFO_PORT; case TABLE_TO_REGIONS_COUNT: return ClusterMetrics.Option.TABLE_TO_REGIONS_COUNT; + case TASKS: return ClusterMetrics.Option.TASKS; // should not reach here default: throw new IllegalArgumentException("Invalid option: " + option); } @@ -188,6 +195,7 @@ public static ClusterStatusProtos.Option toOption(ClusterMetrics.Option option) case SERVERS_NAME: return Option.SERVERS_NAME; case MASTER_INFO_PORT: return ClusterStatusProtos.Option.MASTER_INFO_PORT; case TABLE_TO_REGIONS_COUNT: return ClusterStatusProtos.Option.TABLE_TO_REGIONS_COUNT; + case TASKS: return ClusterStatusProtos.Option.TASKS; // should not reach here default: throw new IllegalArgumentException("Invalid option: " + option); } @@ -231,6 +239,8 @@ public static ClusterMetricsBuilder newBuilder() { private int masterInfoPort; private List serversName = Collections.emptyList(); private Map tableRegionStatesCount = Collections.emptyMap(); + @Nullable + private List masterTasks; private ClusterMetricsBuilder() { } @@ -280,6 +290,10 @@ public ClusterMetricsBuilder setServerNames(List serversName) { this.serversName = serversName; return this; } + public ClusterMetricsBuilder setMasterTasks(List masterTasks) { + this.masterTasks = masterTasks; + return this; + } public ClusterMetricsBuilder setTableRegionStatesCount( Map tableRegionStatesCount) { @@ -300,7 +314,8 @@ public ClusterMetrics build() { balancerOn, masterInfoPort, serversName, - tableRegionStatesCount + tableRegionStatesCount, + masterTasks ); } private static class ClusterMetricsImpl implements ClusterMetrics { @@ -320,6 +335,7 @@ private static class ClusterMetricsImpl implements ClusterMetrics { private final int masterInfoPort; private final List serversName; private final Map tableRegionStatesCount; + private final List masterTasks; ClusterMetricsImpl(String hbaseVersion, List deadServerNames, Map liveServerMetrics, @@ -331,7 +347,8 @@ private static class ClusterMetricsImpl implements ClusterMetrics { Boolean balancerOn, int masterInfoPort, List serversName, - Map tableRegionStatesCount) { + Map tableRegionStatesCount, + List masterTasks) { this.hbaseVersion = hbaseVersion; this.deadServerNames = Preconditions.checkNotNull(deadServerNames); this.liveServerMetrics = Preconditions.checkNotNull(liveServerMetrics); @@ -344,6 +361,7 @@ private static class ClusterMetricsImpl implements ClusterMetrics { this.masterInfoPort = masterInfoPort; this.serversName = serversName; this.tableRegionStatesCount = Preconditions.checkNotNull(tableRegionStatesCount); + this.masterTasks = masterTasks; } @Override @@ -406,6 +424,11 @@ public Map getTableRegionStatesCount() { return Collections.unmodifiableMap(tableRegionStatesCount); } + @Override + public List getMasterTasks() { + return masterTasks; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(1024); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java index 6fdb588a4f37..6a51db08d8c4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java @@ -356,6 +356,11 @@ public Map getTableRegionStatesCount() { return metrics.getTableRegionStatesCount(); } + @Override + public List getMasterTasks() { + return metrics.getMasterTasks(); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(1024); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java index b22d6c4e2446..7b8f713ddb7f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerLoad.java @@ -431,6 +431,11 @@ public long getLastReportTimestamp() { return metrics.getLastReportTimestamp(); } + @Override + public List getTasks() { + return metrics.getTasks(); + } + /** * Originally, this method factored in the effect of requests going to the * server as well. However, this does not interact very well with the current diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetrics.java index 21fad92aa25b..893534cba5f0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetrics.java @@ -114,4 +114,11 @@ default String getVersion() { */ long getLastReportTimestamp(); + /** + * Called directly from clients such as the hbase shell + * @return the active monitored tasks + */ + @Nullable + List getTasks(); + } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java index d93527261d93..89cee9ffaf6f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java @@ -85,6 +85,8 @@ public static ServerMetrics toServerMetrics(ServerName serverName, int versionNu .setReplicationLoadSink(serverLoadPB.hasReplLoadSink() ? ProtobufUtil.toReplicationLoadSink(serverLoadPB.getReplLoadSink()) : null) + .setTasks(serverLoadPB.getTasksList().stream() + .map(ProtobufUtil::getServerTask).collect(Collectors.toList())) .setReportTimestamp(serverLoadPB.getReportEndTime()) .setLastReportTimestamp(serverLoadPB.getReportStartTime()).setVersionNumber(versionNumber) .setVersion(version).build(); @@ -103,19 +105,24 @@ public static ClusterStatusProtos.ServerLoad toServerLoad(ServerMetrics metrics) .setInfoServerPort(metrics.getInfoServerPort()) .setMaxHeapMB((int) metrics.getMaxHeapSize().get(Size.Unit.MEGABYTE)) .setUsedHeapMB((int) metrics.getUsedHeapSize().get(Size.Unit.MEGABYTE)) - .addAllCoprocessors(toCoprocessor(metrics.getCoprocessorNames())).addAllRegionLoads( + .addAllCoprocessors(toCoprocessor(metrics.getCoprocessorNames())) + .addAllRegionLoads( metrics.getRegionMetrics().values().stream().map(RegionMetricsBuilder::toRegionLoad) - .collect(Collectors.toList())).addAllUserLoads( + .collect(Collectors.toList())) + .addAllUserLoads( metrics.getUserMetrics().values().stream().map(UserMetricsBuilder::toUserMetrics) - .collect(Collectors.toList())).addAllReplLoadSource( + .collect(Collectors.toList())) + .addAllReplLoadSource( metrics.getReplicationLoadSourceList().stream() .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList())) + .addAllTasks( + metrics.getTasks().stream().map(ProtobufUtil::toServerTask) + .collect(Collectors.toList())) .setReportStartTime(metrics.getLastReportTimestamp()) .setReportEndTime(metrics.getReportTimestamp()); if (metrics.getReplicationLoadSink() != null) { builder.setReplLoadSink(ProtobufUtil.toReplicationLoadSink(metrics.getReplicationLoadSink())); } - return builder.build(); } @@ -139,6 +146,8 @@ public static ServerMetricsBuilder newBuilder(ServerName sn) { private final Set coprocessorNames = new TreeSet<>(); private long reportTimestamp = EnvironmentEdgeManager.currentTime(); private long lastReportTimestamp = 0; + private final List tasks = new ArrayList<>(); + private ServerMetricsBuilder(ServerName serverName) { this.serverName = serverName; } @@ -213,6 +222,11 @@ public ServerMetricsBuilder setLastReportTimestamp(long value) { return this; } + public ServerMetricsBuilder setTasks(List tasks) { + this.tasks.addAll(tasks); + return this; + } + public ServerMetrics build() { return new ServerMetricsImpl( serverName, @@ -229,7 +243,8 @@ public ServerMetrics build() { coprocessorNames, reportTimestamp, lastReportTimestamp, - userMetrics); + userMetrics, + tasks); } private static class ServerMetricsImpl implements ServerMetrics { @@ -249,12 +264,14 @@ private static class ServerMetricsImpl implements ServerMetrics { private final long reportTimestamp; private final long lastReportTimestamp; private final Map userMetrics; + private final List tasks; ServerMetricsImpl(ServerName serverName, int versionNumber, String version, long requestCountPerSecond, long requestCount, Size usedHeapSize, Size maxHeapSize, int infoServerPort, List sources, ReplicationLoadSink sink, - Map regionStatus, Set coprocessorNames, long reportTimestamp, - long lastReportTimestamp, Map userMetrics) { + Map regionStatus, Set coprocessorNames, + long reportTimestamp, long lastReportTimestamp, Map userMetrics, + List tasks) { this.serverName = Preconditions.checkNotNull(serverName); this.versionNumber = versionNumber; this.version = version; @@ -270,6 +287,7 @@ private static class ServerMetricsImpl implements ServerMetrics { this.coprocessorNames =Preconditions.checkNotNull(coprocessorNames); this.reportTimestamp = reportTimestamp; this.lastReportTimestamp = lastReportTimestamp; + this.tasks = tasks; } @Override @@ -356,6 +374,11 @@ public long getLastReportTimestamp() { return lastReportTimestamp; } + @Override + public List getTasks() { + return tasks; + } + @Override public String toString() { int storeCount = 0; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerTask.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerTask.java new file mode 100644 index 000000000000..e791093e43d7 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerTask.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.yetus.audience.InterfaceAudience; + +/** Information about active monitored server tasks */ +@InterfaceAudience.Public +public interface ServerTask { + + /** Task state */ + enum State { + RUNNING, + WAITING, + COMPLETE, + ABORTED; + } + + /** + * Get the task's description. + * @return the task's description, typically a name + */ + String getDescription(); + + /** + * Get the current status of the task. + * @return the task's current status + */ + String getStatus(); + + /** + * Get the current state of the task. + * @return the task's current state + */ + State getState(); + + /** + * Get the task start time. + * @return the time when the task started, or 0 if it has not started yet + */ + long getStartTime(); + + /** + * Get the task completion time. + * @return the time when the task completed, or 0 if it has not completed yet + */ + long getCompletionTime(); + +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerTaskBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerTaskBuilder.java new file mode 100644 index 000000000000..d4937373789e --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerTaskBuilder.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.yetus.audience.InterfaceAudience; + +/** Builder for information about active monitored server tasks */ +@InterfaceAudience.Private +public final class ServerTaskBuilder { + + public static ServerTaskBuilder newBuilder() { + return new ServerTaskBuilder(); + } + + private String description = ""; + private String status = ""; + private ServerTask.State state = ServerTask.State.RUNNING; + private long startTime; + private long completionTime; + + private ServerTaskBuilder() { } + + private static final class ServerTaskImpl implements ServerTask { + + private final String description; + private final String status; + private final ServerTask.State state; + private final long startTime; + private final long completionTime; + + private ServerTaskImpl(final String description, final String status, + final ServerTask.State state, final long startTime, final long completionTime) { + this.description = description; + this.status = status; + this.state = state; + this.startTime = startTime; + this.completionTime = completionTime; + } + + @Override + public String getDescription() { + return description; + } + + @Override + public String getStatus() { + return status; + } + + @Override + public State getState() { + return state; + } + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public long getCompletionTime() { + return completionTime; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(512); + sb.append(getDescription()); + sb.append(": status="); + sb.append(getStatus()); + sb.append(", state="); + sb.append(getState()); + sb.append(", startTime="); + sb.append(getStartTime()); + sb.append(", completionTime="); + sb.append(getCompletionTime()); + return sb.toString(); + } + + } + + public ServerTaskBuilder setDescription(final String description) { + this.description = description; + return this; + } + + public ServerTaskBuilder setStatus(final String status) { + this.status = status; + return this; + } + + public ServerTaskBuilder setState(final ServerTask.State state) { + this.state = state; + return this; + } + + public ServerTaskBuilder setStartTime(final long startTime) { + this.startTime = startTime; + return this; + } + + public ServerTaskBuilder setCompletionTime(final long completionTime) { + this.completionTime = completionTime; + return this; + } + + public ServerTask build() { + return new ServerTaskImpl(description, status, state, startTime, completionTime); + } + +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 36b106ade1c2..523b22424146 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -66,6 +66,8 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ServerTask; +import org.apache.hadoop.hbase.ServerTaskBuilder; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.BalanceResponse; @@ -3792,4 +3794,24 @@ public static BalanceResponse toBalanceResponse(MasterProtos.BalanceResponse res .build(); } + public static ServerTask getServerTask(ClusterStatusProtos.ServerTask task) { + return ServerTaskBuilder.newBuilder() + .setDescription(task.getDescription()) + .setStatus(task.getStatus()) + .setState(ServerTask.State.valueOf(task.getState().name())) + .setStartTime(task.getStartTime()) + .setCompletionTime(task.getCompletionTime()) + .build(); + } + + public static ClusterStatusProtos.ServerTask toServerTask(ServerTask task) { + return ClusterStatusProtos.ServerTask.newBuilder() + .setDescription(task.getDescription()) + .setStatus(task.getStatus()) + .setState(ClusterStatusProtos.ServerTask.State.valueOf(task.getState().name())) + .setStartTime(task.getStartTime()) + .setCompletionTime(task.getCompletionTime()) + .build(); + } + } diff --git a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto index acff682769bc..4957d7ca8017 100644 --- a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto @@ -229,6 +229,21 @@ message ReplicationLoadSource { optional uint64 oPsShipped = 12; } +message ServerTask { + required string description = 1; + required string status = 2; + required State state = 3; + optional uint64 startTime = 4; + optional uint64 completionTime = 5; + + enum State { + RUNNING = 0; + WAITING = 1; + COMPLETE = 2; + ABORTED = 3; + } +} + message ServerLoad { /** Number of requests since last report. */ optional uint64 number_of_requests = 1; @@ -285,6 +300,11 @@ message ServerLoad { * The metrics for each user on this region server */ repeated UserLoad userLoads = 12; + + /** + * The active monitored tasks + */ + repeated ServerTask tasks = 15; /* 15 here to stay in sync with master branch */ } message LiveServerInfo { @@ -318,6 +338,7 @@ message ClusterStatus { optional int32 master_info_port = 10 [default = -1]; repeated ServerName servers_name = 11; repeated TableRegionStatesCount table_region_states_count = 12; + repeated ServerTask master_tasks = 13; } enum Option { @@ -333,4 +354,5 @@ enum Option { MASTER_INFO_PORT = 9; SERVERS_NAME = 10; TABLE_TO_REGIONS_COUNT = 11; + TASKS = 12; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 7820c2ac5c06..a91fe016e7b1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -79,6 +79,8 @@ import org.apache.hadoop.hbase.ReplicationPeerNotFoundException; import org.apache.hadoop.hbase.ServerMetrics; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ServerTask; +import org.apache.hadoop.hbase.ServerTaskBuilder; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; @@ -2712,16 +2714,39 @@ public ClusterMetrics getClusterMetricsWithoutCoprocessor(EnumSet