diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index 2eb16e76497c..4dd85ef0170e 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -144,14 +144,6 @@ 0644 - - ${project.basedir}/../hbase-rsgroup/target/ - lib - - ${rsgroup.test.jar} - - 0644 - ${project.basedir}/../hbase-mapreduce/target/ lib diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml index 91d374987a62..6a21b9208731 100644 --- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml @@ -52,7 +52,6 @@ org.apache.hbase:hbase-protocol-shaded org.apache.hbase:hbase-replication org.apache.hbase:hbase-rest - org.apache.hbase:hbase-rsgroup org.apache.hbase:hbase-server org.apache.hbase:hbase-shell org.apache.hbase:hbase-testing-util diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index ac0695b35ddd..7989c8935874 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -1768,23 +1768,36 @@ public static ServerName toServerName(final byte [] data) throws Deserialization return ServerName.valueOf(hostname, port, -1L); } + public static HBaseProtos.TimeRange toTimeRange(TimeRange timeRange) { + if (timeRange == null) { + timeRange = TimeRange.allTime(); + } + return HBaseProtos.TimeRange.newBuilder().setFrom(timeRange.getMin()).setTo(timeRange.getMax()) + .build(); + } + public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) { RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName()); - for(HBaseProtos.ServerName el: proto.getServersList()) { + for (HBaseProtos.ServerName el : proto.getServersList()) { RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort())); } - for(HBaseProtos.TableName pTableName: proto.getTablesList()) { + for (HBaseProtos.TableName pTableName : proto.getTablesList()) { RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName)); } return RSGroupInfo; } - public static HBaseProtos.TimeRange toTimeRange(TimeRange timeRange) { - if (timeRange == null) { - timeRange = TimeRange.allTime(); + public static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) { + List tables = new ArrayList<>(pojo.getTables().size()); + for (TableName arg : pojo.getTables()) { + tables.add(ProtobufUtil.toProtoTableName(arg)); + } + List hostports = new ArrayList<>(pojo.getServers().size()); + for (Address el : pojo.getServers()) { + hostports.add(HBaseProtos.ServerName.newBuilder().setHostName(el.getHostname()) + .setPort(el.getPort()).build()); } - return HBaseProtos.TimeRange.newBuilder().setFrom(timeRange.getMin()) - .setTo(timeRange.getMax()) - .build(); + return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()).addAllServers(hostports) + .addAllTables(tables).build(); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index f31d8e418662..4f6bc4161fb6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -91,6 +91,7 @@ import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.io.TimeRange; +import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.protobuf.ProtobufMagic; import org.apache.hadoop.hbase.protobuf.ProtobufMessageConverter; import org.apache.hadoop.hbase.quotas.QuotaScope; @@ -99,6 +100,7 @@ import org.apache.hadoop.hbase.quotas.ThrottleType; import org.apache.hadoop.hbase.replication.ReplicationLoadSink; import org.apache.hadoop.hbase.replication.ReplicationLoadSource; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.security.visibility.Authorizations; import org.apache.hadoop.hbase.security.visibility.CellVisibility; import org.apache.hadoop.hbase.util.Addressing; @@ -176,6 +178,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MajorCompactionTimestampResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; @@ -3298,4 +3301,29 @@ public static Set toCompactedStoreFiles(byte[] bytes) throws IOException } return Collections.emptySet(); } + + public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) { + RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName()); + for (HBaseProtos.ServerName el : proto.getServersList()) { + RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort())); + } + for (HBaseProtos.TableName pTableName : proto.getTablesList()) { + RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName)); + } + return RSGroupInfo; + } + + public static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) { + List tables = new ArrayList<>(pojo.getTables().size()); + for (TableName arg : pojo.getTables()) { + tables.add(ProtobufUtil.toProtoTableName(arg)); + } + List hostports = new ArrayList<>(pojo.getServers().size()); + for (Address el : pojo.getServers()) { + hostports.add(HBaseProtos.ServerName.newBuilder().setHostName(el.getHostname()) + .setPort(el.getPort()).build()); + } + return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()).addAllServers(hostports) + .addAllTables(tables).build(); + } } diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 061b7b2800f3..2b4ef475aa01 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -181,16 +181,6 @@ test-jar test - - org.apache.hbase - hbase-rsgroup - - - org.apache.hbase - hbase-rsgroup - test-jar - test - org.apache.hbase hbase-server diff --git a/hbase-rsgroup/src/test/resources/hbase-site.xml b/hbase-protocol-shaded/src/main/protobuf/RSGroup.proto similarity index 67% rename from hbase-rsgroup/src/test/resources/hbase-site.xml rename to hbase-protocol-shaded/src/main/protobuf/RSGroup.proto index 99d2ab8d1fbb..ede2b13cb5cc 100644 --- a/hbase-rsgroup/src/test/resources/hbase-site.xml +++ b/hbase-protocol-shaded/src/main/protobuf/RSGroup.proto @@ -1,8 +1,4 @@ - - - - - - hbase.defaults.for.version.skip - true - - - hbase.hconnection.threads.keepalivetime - 3 - - + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated"; +option java_outer_classname = "RSGroupProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; + +message RSGroupInfo { + required string name = 1; + repeated ServerName servers = 4; + repeated TableName tables = 3; +} diff --git a/hbase-protocol-shaded/src/main/protobuf/RSGroupAdmin.proto b/hbase-protocol-shaded/src/main/protobuf/RSGroupAdmin.proto new file mode 100644 index 000000000000..1db713677f2c --- /dev/null +++ b/hbase-protocol-shaded/src/main/protobuf/RSGroupAdmin.proto @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hbase.pb; + +option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated"; +option java_outer_classname = "RSGroupAdminProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +import "HBase.proto"; +import "RSGroup.proto"; + +/** Group level protobufs */ + +message ListTablesOfRSGroupRequest { + required string r_s_group_name = 1; +} + +message ListTablesOfRSGroupResponse { + repeated TableName table_name = 1; +} + +message GetRSGroupInfoRequest { + required string r_s_group_name = 1; +} + +message GetRSGroupInfoResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +message GetRSGroupInfoOfTableRequest { + required TableName table_name = 1; +} + +message GetRSGroupInfoOfTableResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +message MoveServersRequest { + required string target_group = 1; + repeated ServerName servers = 3; +} + +message MoveServersResponse { +} + +message MoveTablesRequest { + required string target_group = 1; + repeated TableName table_name = 2; +} + +message MoveTablesResponse { +} + +message AddRSGroupRequest { + required string r_s_group_name = 1; +} + +message AddRSGroupResponse { +} + +message RemoveRSGroupRequest { + required string r_s_group_name = 1; +} + +message RemoveRSGroupResponse { +} + +message BalanceRSGroupRequest { + required string r_s_group_name = 1; +} + +message BalanceRSGroupResponse { + required bool balanceRan = 1; +} + +message ListRSGroupInfosRequest { +} + +message ListRSGroupInfosResponse { + repeated RSGroupInfo r_s_group_info = 1; +} + +message GetRSGroupInfoOfServerRequest { + required ServerName server = 2; +} + +message GetRSGroupInfoOfServerResponse { + optional RSGroupInfo r_s_group_info = 1; +} + +message MoveServersAndTablesRequest { + required string target_group = 1; + repeated ServerName servers = 2; + repeated TableName table_name = 3; +} + +message MoveServersAndTablesResponse { +} + +message RemoveServersRequest { + repeated ServerName servers = 1; +} + +message RemoveServersResponse { +} + +service RSGroupAdminService { + rpc GetRSGroupInfo(GetRSGroupInfoRequest) + returns (GetRSGroupInfoResponse); + + rpc GetRSGroupInfoOfTable(GetRSGroupInfoOfTableRequest) + returns (GetRSGroupInfoOfTableResponse); + + rpc GetRSGroupInfoOfServer(GetRSGroupInfoOfServerRequest) + returns (GetRSGroupInfoOfServerResponse); + + rpc MoveServers(MoveServersRequest) + returns (MoveServersResponse); + + rpc MoveTables(MoveTablesRequest) + returns (MoveTablesResponse); + + rpc AddRSGroup(AddRSGroupRequest) + returns (AddRSGroupResponse); + + rpc RemoveRSGroup(RemoveRSGroupRequest) + returns (RemoveRSGroupResponse); + + rpc BalanceRSGroup(BalanceRSGroupRequest) + returns (BalanceRSGroupResponse); + + rpc ListRSGroupInfos(ListRSGroupInfosRequest) + returns (ListRSGroupInfosResponse); + + rpc MoveServersAndTables(MoveServersAndTablesRequest) + returns (MoveServersAndTablesResponse); + + rpc RemoveServers(RemoveServersRequest) + returns (RemoveServersResponse); +} diff --git a/hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto b/hbase-protocol/src/main/protobuf/RSGroupAdmin.proto similarity index 100% rename from hbase-rsgroup/src/main/protobuf/RSGroupAdmin.proto rename to hbase-protocol/src/main/protobuf/RSGroupAdmin.proto diff --git a/hbase-rsgroup/README.txt b/hbase-rsgroup/README.txt deleted file mode 100644 index b24aee650603..000000000000 --- a/hbase-rsgroup/README.txt +++ /dev/null @@ -1,13 +0,0 @@ -ON PROTOBUFS -This maven module has protobuf definition files ('.protos') used by hbase -Coprocessor Endpoints that ship with hbase core including tests. Coprocessor -Endpoints are meant to be standalone, independent code not reliant on hbase -internals. They define their Service using protobuf. The protobuf version -they use can be distinct from that used by HBase internally since HBase started -shading its protobuf references. Endpoints have no access to the shaded protobuf -hbase uses. They do have access to the content of hbase-protocol -- the -.protos found in here -- but avoid using as much of this as you can as it is -liable to change. - -Generation of java files from protobuf .proto files included here is done as -part of the build. diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml deleted file mode 100644 index b494a9afde7c..000000000000 --- a/hbase-rsgroup/pom.xml +++ /dev/null @@ -1,278 +0,0 @@ - - - - 4.0.0 - - hbase-build-configuration - org.apache.hbase - 3.0.0-SNAPSHOT - ../hbase-build-configuration - - hbase-rsgroup - Apache HBase - RSGroup - Regionserver Groups for HBase - - - - - maven-assembly-plugin - - true - - - - - org.apache.maven.plugins - maven-source-plugin - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - compile-protoc - generate-sources - - compile - - - - ${basedir}/../hbase-protocol/src/main/protobuf - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - true - - - - net.revelc.code - warbucks-maven-plugin - - - - - - - org.apache.hbase - hbase-annotations - test-jar - test - - - org.apache.hbase - hbase-client - - - org.apache.hbase - hbase-server - - - org.apache.hbase - hbase-common - - - org.apache.hbase - hbase-procedure - - - org.apache.hbase - hbase-protocol - - - org.apache.hbase - hbase-protocol-shaded - - - org.apache.hbase - hbase-testing-util - test - - - - org.apache.commons - commons-lang3 - - - org.slf4j - slf4j-api - - - org.apache.hbase.thirdparty - hbase-shaded-miscellaneous - - - com.google.protobuf - protobuf-java - - - org.apache.zookeeper - zookeeper - - - log4j - log4j - test - - - org.mockito - mockito-core - test - - - junit - junit - test - - - - - - skipRSGroupTests - - - skipRSGroupTests - - - - true - true - - - - - hadoop-2.0 - - - - - !hadoop.profile - - - - - com.github.stephenc.findbugs - findbugs-annotations - true - - - org.apache.hadoop - hadoop-common - - - net.java.dev.jets3t - jets3t - - - javax.servlet.jsp - jsp-api - - - org.mortbay.jetty - jetty - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - javax.servlet - servlet-api - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - com.google.code.findbugs - jsr305 - - - - - - - - hadoop-3.0 - - - hadoop.profile - 3.0 - - - - 3.0-SNAPSHOT - - - - org.apache.hadoop - hadoop-common - - - - - eclipse-specific - - - m2e.version - - - - - - - - org.eclipse.m2e - lifecycle-mapping - - - - - - - - - - - - - diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java deleted file mode 100644 index 090ac6e907e8..000000000000 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java +++ /dev/null @@ -1,561 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.rsgroup; - -import com.google.protobuf.RpcCallback; -import com.google.protobuf.RpcController; -import com.google.protobuf.Service; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.PleaseHoldException; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.constraint.ConstraintException; -import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor; -import org.apache.hadoop.hbase.coprocessor.HasMasterServices; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.MasterObserver; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; -import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RSGroupAdminService; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersResponse; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessChecker; -import org.apache.hadoop.hbase.security.access.Permission.Action; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hbase.thirdparty.com.google.common.collect.Sets; - -// TODO: Encapsulate MasterObserver functions into separate subclass. -@CoreCoprocessor -@InterfaceAudience.Private -public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { - private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminEndpoint.class); - - private MasterServices master = null; - // Only instance of RSGroupInfoManager. RSGroup aware load balancers ask for this instance on - // their setup. - private RSGroupInfoManager groupInfoManager; - private RSGroupAdminServer groupAdminServer; - private final RSGroupAdminService groupAdminService = new RSGroupAdminServiceImpl(); - private AccessChecker accessChecker; - - /** Provider for mapping principal names to Users */ - private UserProvider userProvider; - - @Override - public void start(CoprocessorEnvironment env) throws IOException { - if (!(env instanceof HasMasterServices)) { - throw new IOException("Does not implement HMasterServices"); - } - - master = ((HasMasterServices)env).getMasterServices(); - groupInfoManager = RSGroupInfoManagerImpl.getInstance(master); - groupAdminServer = new RSGroupAdminServer(master, groupInfoManager); - Class clazz = - master.getConfiguration().getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, null); - if (!RSGroupableBalancer.class.isAssignableFrom(clazz)) { - throw new IOException("Configured balancer does not support RegionServer groups."); - } - accessChecker = ((HasMasterServices) env).getMasterServices().getAccessChecker(); - - // set the user-provider. - this.userProvider = UserProvider.instantiate(env.getConfiguration()); - } - - @Override - public void stop(CoprocessorEnvironment env) { - } - - @Override - public Iterable getServices() { - return Collections.singleton(groupAdminService); - } - - @Override - public Optional getMasterObserver() { - return Optional.of(this); - } - - RSGroupInfoManager getGroupInfoManager() { - return groupInfoManager; - } - - /** - * Implementation of RSGroupAdminService defined in RSGroupAdmin.proto. - * This class calls {@link RSGroupAdminServer} for actual work, converts result to protocol - * buffer response, handles exceptions if any occurred and then calls the {@code RpcCallback} with - * the response. - */ - private class RSGroupAdminServiceImpl extends RSGroupAdminProtos.RSGroupAdminService { - @Override - public void getRSGroupInfo(RpcController controller, - GetRSGroupInfoRequest request, RpcCallback done) { - GetRSGroupInfoResponse.Builder builder = GetRSGroupInfoResponse.newBuilder(); - String groupName = request.getRSGroupName(); - LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, group=" - + groupName); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preGetRSGroupInfo(groupName); - } - checkPermission("getRSGroupInfo"); - RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); - if (rsGroupInfo != null) { - builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(rsGroupInfo)); - } - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postGetRSGroupInfo(groupName); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void getRSGroupInfoOfTable(RpcController controller, - GetRSGroupInfoOfTableRequest request, RpcCallback done) { - GetRSGroupInfoOfTableResponse.Builder builder = GetRSGroupInfoOfTableResponse.newBuilder(); - TableName tableName = ProtobufUtil.toTableName(request.getTableName()); - LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, table=" - + tableName); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preGetRSGroupInfoOfTable(tableName); - } - checkPermission("getRSGroupInfoOfTable"); - RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupInfoOfTable(tableName); - if (RSGroupInfo != null) { - builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo)); - } - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postGetRSGroupInfoOfTable(tableName); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveServers(RpcController controller, MoveServersRequest request, - RpcCallback done) { - MoveServersResponse.Builder builder = MoveServersResponse.newBuilder(); - Set
hostPorts = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); - } - LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts +" to rsgroup " - + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveServers(hostPorts, request.getTargetGroup()); - } - checkPermission("moveServers"); - groupAdminServer.moveServers(hostPorts, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveServers(hostPorts, request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveTables(RpcController controller, MoveTablesRequest request, - RpcCallback done) { - MoveTablesResponse.Builder builder = MoveTablesResponse.newBuilder(); - Set tables = new HashSet<>(request.getTableNameList().size()); - for (HBaseProtos.TableName tableName : request.getTableNameList()) { - tables.add(ProtobufUtil.toTableName(tableName)); - } - LOG.info(master.getClientIdAuditPrefix() + " move tables " + tables +" to rsgroup " - + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveTables(tables, request.getTargetGroup()); - } - checkPermission("moveTables"); - groupAdminServer.moveTables(tables, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveTables(tables, request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void addRSGroup(RpcController controller, AddRSGroupRequest request, - RpcCallback done) { - AddRSGroupResponse.Builder builder = AddRSGroupResponse.newBuilder(); - LOG.info(master.getClientIdAuditPrefix() + " add rsgroup " + request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preAddRSGroup(request.getRSGroupName()); - } - checkPermission("addRSGroup"); - groupAdminServer.addRSGroup(request.getRSGroupName()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postAddRSGroup(request.getRSGroupName()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void removeRSGroup(RpcController controller, - RemoveRSGroupRequest request, RpcCallback done) { - RemoveRSGroupResponse.Builder builder = - RemoveRSGroupResponse.newBuilder(); - LOG.info(master.getClientIdAuditPrefix() + " remove rsgroup " + request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preRemoveRSGroup(request.getRSGroupName()); - } - checkPermission("removeRSGroup"); - groupAdminServer.removeRSGroup(request.getRSGroupName()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postRemoveRSGroup(request.getRSGroupName()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void balanceRSGroup(RpcController controller, - BalanceRSGroupRequest request, RpcCallback done) { - BalanceRSGroupResponse.Builder builder = BalanceRSGroupResponse.newBuilder(); - LOG.info(master.getClientIdAuditPrefix() + " balance rsgroup, group=" + - request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preBalanceRSGroup(request.getRSGroupName()); - } - checkPermission("balanceRSGroup"); - boolean balancerRan = groupAdminServer.balanceRSGroup(request.getRSGroupName()); - builder.setBalanceRan(balancerRan); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postBalanceRSGroup(request.getRSGroupName(), - balancerRan); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - builder.setBalanceRan(false); - } - done.run(builder.build()); - } - - @Override - public void listRSGroupInfos(RpcController controller, - ListRSGroupInfosRequest request, RpcCallback done) { - ListRSGroupInfosResponse.Builder builder = ListRSGroupInfosResponse.newBuilder(); - LOG.info(master.getClientIdAuditPrefix() + " list rsgroup"); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preListRSGroups(); - } - checkPermission("listRSGroup"); - for (RSGroupInfo RSGroupInfo : groupAdminServer.listRSGroups()) { - builder.addRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo)); - } - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postListRSGroups(); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void getRSGroupInfoOfServer(RpcController controller, - GetRSGroupInfoOfServerRequest request, RpcCallback done) { - GetRSGroupInfoOfServerResponse.Builder builder = GetRSGroupInfoOfServerResponse.newBuilder(); - Address hp = Address.fromParts(request.getServer().getHostName(), - request.getServer().getPort()); - LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, server=" - + hp); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preGetRSGroupInfoOfServer(hp); - } - checkPermission("getRSGroupInfoOfServer"); - RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp); - if (info != null) { - builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(info)); - } - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postGetRSGroupInfoOfServer(hp); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveServersAndTables(RpcController controller, - MoveServersAndTablesRequest request, RpcCallback done) { - MoveServersAndTablesResponse.Builder builder = MoveServersAndTablesResponse.newBuilder(); - Set
hostPorts = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); - } - Set tables = new HashSet<>(request.getTableNameList().size()); - for (HBaseProtos.TableName tableName : request.getTableNameList()) { - tables.add(ProtobufUtil.toTableName(tableName)); - } - LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts - + " and tables " + tables + " to rsgroup" + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveServersAndTables(hostPorts, tables, - request.getTargetGroup()); - } - checkPermission("moveServersAndTables"); - groupAdminServer.moveServersAndTables(hostPorts, tables, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveServersAndTables(hostPorts, tables, - request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void removeServers(RpcController controller, - RemoveServersRequest request, - RpcCallback done) { - RemoveServersResponse.Builder builder = - RemoveServersResponse.newBuilder(); - Set
servers = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - servers.add(Address.fromParts(el.getHostName(), el.getPort())); - } - LOG.info(master.getClientIdAuditPrefix() - + " remove decommissioned servers from rsgroup: " + servers); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preRemoveServers(servers); - } - checkPermission("removeServers"); - groupAdminServer.removeServers(servers); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postRemoveServers(servers); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - } - - boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { - String groupName; - try { - groupName = - master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) - .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (groupName == null) { - groupName = RSGroupInfo.DEFAULT_GROUP; - } - } catch (MasterNotRunningException | PleaseHoldException e) { - LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + - RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); - groupName = RSGroupInfo.DEFAULT_GROUP; - } - - RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); - if (rsGroupInfo == null) { - throw new ConstraintException( - "Default RSGroup (" + groupName + ") for this table's " + "namespace does not exist."); - } - - for (ServerName onlineServer : master.getServerManager().createDestinationServersList()) { - if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) { - return true; - } - } - return false; - } - - void assignTableToGroup(TableDescriptor desc) throws IOException { - String groupName = - master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) - .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (groupName == null) { - groupName = RSGroupInfo.DEFAULT_GROUP; - } - RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); - if (rsGroupInfo == null) { - throw new ConstraintException("Default RSGroup (" + groupName + ") for this table's " - + "namespace does not exist."); - } - if (!rsGroupInfo.containsTable(desc.getTableName())) { - LOG.debug("Pre-moving table " + desc.getTableName() + " to RSGroup " + groupName); - groupAdminServer.moveTables(Sets.newHashSet(desc.getTableName()), groupName); - } - } - - ///////////////////////////////////////////////////////////////////////////// - // MasterObserver overrides - ///////////////////////////////////////////////////////////////////////////// - - @Override - public void preCreateTableAction( - final ObserverContext ctx, - final TableDescriptor desc, - final RegionInfo[] regions) throws IOException { - if (!desc.getTableName().isSystemTable() && !rsgroupHasServersOnline(desc)) { - throw new HBaseIOException("No online servers in the rsgroup, which table " + - desc.getTableName().getNameAsString() + " belongs to"); - } - } - - // Assign table to default RSGroup. - @Override - public void postCreateTable(ObserverContext ctx, - TableDescriptor desc, RegionInfo[] regions) throws IOException { - assignTableToGroup(desc); - } - - // Remove table from its RSGroup. - @Override - public void postDeleteTable(ObserverContext ctx, - TableName tableName) throws IOException { - try { - RSGroupInfo group = groupAdminServer.getRSGroupInfoOfTable(tableName); - if (group != null) { - LOG.debug(String.format("Removing deleted table '%s' from rsgroup '%s'", tableName, - group.getName())); - groupAdminServer.moveTables(Sets.newHashSet(tableName), null); - } - } catch (IOException ex) { - LOG.debug("Failed to perform RSGroup information cleanup for table: " + tableName, ex); - } - } - - @Override - public void preCreateNamespace(ObserverContext ctx, - NamespaceDescriptor ns) throws IOException { - String group = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if(group != null && groupAdminServer.getRSGroupInfo(group) == null) { - throw new ConstraintException("Region server group "+group+" does not exit"); - } - } - - @Override - public void preModifyNamespace(ObserverContext ctx, - NamespaceDescriptor currentNsDesc, NamespaceDescriptor newNsDesc) throws IOException { - preCreateNamespace(ctx, newNsDesc); - } - - @Override - public void preCloneSnapshot(ObserverContext ctx, - SnapshotDescription snapshot, TableDescriptor desc) throws IOException { - assignTableToGroup(desc); - } - - @Override - public void postClearDeadServers(ObserverContext ctx, - List servers, List notClearedServers) - throws IOException { - Set
clearedServer = servers.stream(). - filter(server -> !notClearedServers.contains(server)). - map(ServerName::getAddress). - collect(Collectors.toSet()); - if(!clearedServer.isEmpty()) { - groupAdminServer.removeServers(clearedServer); - } - } - - public void checkPermission(String request) throws IOException { - accessChecker.requirePermission(getActiveUser(), request, null, Action.ADMIN); - } - - /** - * Returns the active user to which authorization checks should be applied. - * If we are in the context of an RPC call, the remote user is used, - * otherwise the currently logged in user is used. - */ - private User getActiveUser() throws IOException { - // for non-rpc handling, fallback to system user - Optional optionalUser = RpcServer.getRequestUser(); - if (optionalUser.isPresent()) { - return optionalUser.get(); - } - return userProvider.getCurrent(); - } -} diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java deleted file mode 100644 index 398e8a4008c1..000000000000 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.rsgroup; - -import java.io.IOException; -import java.util.List; -import java.util.Set; - -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Interface used to manage RSGroupInfo storage. An implementation - * has the option to support offline mode. - * See {@link RSGroupBasedLoadBalancer} - */ -@InterfaceAudience.Private -public interface RSGroupInfoManager { - - String REASSIGN_WAIT_INTERVAL_KEY = "hbase.rsgroup.reassign.wait"; - long DEFAULT_REASSIGN_WAIT_INTERVAL = 30 * 1000L; - - //Assigned before user tables - TableName RSGROUP_TABLE_NAME = - TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "rsgroup"); - String rsGroupZNode = "rsgroup"; - byte[] META_FAMILY_BYTES = Bytes.toBytes("m"); - byte[] META_QUALIFIER_BYTES = Bytes.toBytes("i"); - byte[] ROW_KEY = {0}; - - void start(); - - /** - * Add given RSGroupInfo to existing list of group infos. - */ - void addRSGroup(RSGroupInfo rsGroupInfo) throws IOException; - - /** - * Remove a region server group. - */ - void removeRSGroup(String groupName) throws IOException; - - /** - * Move servers to a new group. - * @param servers list of servers, must be part of the same group - * @param srcGroup groupName being moved from - * @param dstGroup groupName being moved to - * @return Set of servers moved (May be a subset of {@code servers}). - */ - Set
moveServers(Set
servers, String srcGroup, String dstGroup) - throws IOException; - - /** - * Gets the group info of server. - */ - RSGroupInfo getRSGroupOfServer(Address serverHostPort) throws IOException; - - /** - * Gets {@code RSGroupInfo} for the given group name. - */ - RSGroupInfo getRSGroup(String groupName) throws IOException; - - /** - * Get the group membership of a table - */ - String getRSGroupOfTable(TableName tableName) throws IOException; - - /** - * Set the group membership of a set of tables - * - * @param tableNames set of tables to move - * @param groupName name of group of tables to move to - */ - void moveTables(Set tableNames, String groupName) throws IOException; - - /** - * List the existing {@code RSGroupInfo}s. - */ - List listRSGroups() throws IOException; - - /** - * Refresh/reload the group information from the persistent store - */ - void refresh() throws IOException; - - /** - * Whether the manager is able to fully return group metadata - * - * @return whether the manager is in online mode - */ - boolean isOnline(); - - /** - * Move servers and tables to a new group. - * @param servers list of servers, must be part of the same group - * @param tables set of tables to move - * @param srcGroup groupName being moved from - * @param dstGroup groupName being moved to - */ - void moveServersAndTables(Set
servers, Set tables, - String srcGroup, String dstGroup) throws IOException; - - /** - * Remove decommissioned servers from rsgroup - * @param servers set of servers to remove - */ - void removeServers(Set
servers) throws IOException; -} diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupProtobufUtil.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupProtobufUtil.java deleted file mode 100644 index 56e35e76197c..000000000000 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupProtobufUtil.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.rsgroup; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; -import org.apache.yetus.audience.InterfaceAudience; - -@InterfaceAudience.Private -final class RSGroupProtobufUtil { - private RSGroupProtobufUtil() { - } - - static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) { - RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName()); - for(HBaseProtos.ServerName el: proto.getServersList()) { - RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort())); - } - for(HBaseProtos.TableName pTableName: proto.getTablesList()) { - RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName)); - } - return RSGroupInfo; - } - - static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) { - List tables = new ArrayList<>(pojo.getTables().size()); - for(TableName arg: pojo.getTables()) { - tables.add(ProtobufUtil.toProtoTableName(arg)); - } - List hostports = new ArrayList<>(pojo.getServers().size()); - for(Address el: pojo.getServers()) { - hostports.add(HBaseProtos.ServerName.newBuilder() - .setHostName(el.getHostname()) - .setPort(el.getPort()) - .build()); - } - return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()) - .addAllServers(hostports) - .addAllTables(tables).build(); - } -} diff --git a/hbase-rsgroup/src/test/resources/log4j.properties b/hbase-rsgroup/src/test/resources/log4j.properties deleted file mode 100644 index c322699ced24..000000000000 --- a/hbase-rsgroup/src/test/resources/log4j.properties +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hbase.root.logger=INFO,console -hbase.log.dir=. -hbase.log.file=hbase.log - -# Define the root logger to the system property "hbase.root.logger". -log4j.rootLogger=${hbase.root.logger} - -# Logging Threshold -log4j.threshold=ALL - -# -# Daily Rolling File Appender -# -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout -# Debugging Pattern format -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n - -# Custom Logging levels - -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG - -log4j.logger.org.apache.hadoop=WARN -log4j.logger.org.apache.zookeeper=ERROR -log4j.logger.org.apache.hadoop.hbase=DEBUG - -#These settings are workarounds against spurious logs from the minicluster. -#See HBASE-4709 -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN -log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN -log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN -# Enable this to get detailed connection error/retry logging. -# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 31587350bf56..984cad8d7371 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -186,6 +186,7 @@ import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner; import org.apache.hadoop.hbase.replication.master.ReplicationPeerConfigUpgrader; import org.apache.hadoop.hbase.replication.regionserver.ReplicationStatus; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.SecurityConstants; import org.apache.hadoop.hbase.security.UserProvider; @@ -350,6 +351,8 @@ public void run() { // manager of assignment nodes in zookeeper private AssignmentManager assignmentManager; + private RSGroupInfoManager rsGroupInfoManager; + // manager of replication private ReplicationPeerManager replicationPeerManager; @@ -770,6 +773,9 @@ protected void initializeZKBasedSystemTrackers() this.splitOrMergeTracker = new SplitOrMergeTracker(zooKeeper, conf, this); this.splitOrMergeTracker.start(); + this.rsGroupInfoManager = new RSGroupInfoManager(this); + this.rsGroupInfoManager.init(); + this.replicationPeerManager = ReplicationPeerManager.create(zooKeeper, conf); this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this, this.serverManager); @@ -3756,4 +3762,9 @@ public Map getWalGroupsReplicationStatus() { } return super.getWalGroupsReplicationStatus(); } + + @Override + public RSGroupInfoManager getRSRSGroupInfoManager() { + return this.rsGroupInfoManager; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 41cec5cfb232..84713afb526a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.SyncReplicationState; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager; import org.apache.hadoop.hbase.security.access.AccessChecker; import org.apache.hadoop.hbase.security.access.ZKPermissionWatcher; import org.apache.yetus.audience.InterfaceAudience; @@ -537,4 +538,8 @@ default SplitWALManager getSplitWALManager(){ */ List executeRegionPlansWithThrottling(List plans); + /** + * @return the {@link RSGroupInfoManager} + */ + RSGroupInfoManager getRSRSGroupInfoManager(); } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java similarity index 97% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java index e8a141064381..e7ab7f23e805 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java @@ -18,12 +18,10 @@ package org.apache.hadoop.hbase.rsgroup; import com.google.protobuf.ServiceException; - import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; - import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.Admin; @@ -70,7 +68,7 @@ public RSGroupInfo getRSGroupInfo(String groupName) throws IOException { GetRSGroupInfoResponse resp = stub.getRSGroupInfo(null, GetRSGroupInfoRequest.newBuilder().setRSGroupName(groupName).build()); if(resp.hasRSGroupInfo()) { - return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); + return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); } return null; } catch (ServiceException e) { @@ -85,7 +83,7 @@ public RSGroupInfo getRSGroupInfoOfTable(TableName tableName) throws IOException try { GetRSGroupInfoOfTableResponse resp = stub.getRSGroupInfoOfTable(null, request); if (resp.hasRSGroupInfo()) { - return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); + return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); } return null; } catch (ServiceException e) { @@ -167,7 +165,7 @@ public List listRSGroups() throws IOException { ListRSGroupInfosRequest.getDefaultInstance()).getRSGroupInfoList(); List result = new ArrayList<>(resp.size()); for(RSGroupProtos.RSGroupInfo entry : resp) { - result.add(RSGroupProtobufUtil.toGroupInfo(entry)); + result.add(ProtobufUtil.toGroupInfo(entry)); } return result; } catch (ServiceException e) { @@ -186,7 +184,7 @@ public RSGroupInfo getRSGroupOfServer(Address hostPort) throws IOException { try { GetRSGroupInfoOfServerResponse resp = stub.getRSGroupInfoOfServer(null, request); if (resp.hasRSGroupInfo()) { - return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); + return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo()); } return null; } catch (ServiceException e) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java new file mode 100644 index 000000000000..7c03f6b17558 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import com.google.protobuf.Service; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.PleaseHoldException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.constraint.ConstraintException; +import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor; +import org.apache.hadoop.hbase.coprocessor.HasMasterServices; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.AccessChecker; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; + +// TODO: Encapsulate MasterObserver functions into separate subclass. +@CoreCoprocessor +@InterfaceAudience.Private +public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { + static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminEndpoint.class); + + private MasterServices master; + // Only instance of RSGroupInfoManager. RSGroup aware load balancers ask for this instance on + // their setup. + private RSGroupInfoManager groupInfoManager; + private RSGroupAdminServer groupAdminServer; + private RSGroupAdminServiceImpl groupAdminService = new RSGroupAdminServiceImpl(); + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + if (!(env instanceof HasMasterServices)) { + throw new IOException("Does not implement HMasterServices"); + } + + master = ((HasMasterServices) env).getMasterServices(); + groupInfoManager = master.getRSRSGroupInfoManager(); + groupAdminServer = new RSGroupAdminServer(master, groupInfoManager); + Class clazz = + master.getConfiguration().getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, null); + if (!RSGroupableBalancer.class.isAssignableFrom(clazz)) { + throw new IOException("Configured balancer does not support RegionServer groups."); + } + AccessChecker accessChecker = ((HasMasterServices) env).getMasterServices().getAccessChecker(); + + // set the user-provider. + UserProvider userProvider = UserProvider.instantiate(env.getConfiguration()); + groupAdminService.initialize(master, groupAdminServer, accessChecker, userProvider); + } + + @Override + public void stop(CoprocessorEnvironment env) { + } + + @Override + public Iterable getServices() { + return Collections.singleton(groupAdminService); + } + + @Override + public Optional getMasterObserver() { + return Optional.of(this); + } + + RSGroupInfoManager getGroupInfoManager() { + return groupInfoManager; + } + + @VisibleForTesting + RSGroupAdminServiceImpl getGroupAdminService() { + return groupAdminService; + } + + private void assignTableToGroup(TableDescriptor desc) throws IOException { + String groupName = + master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) + .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (groupName == null) { + groupName = RSGroupInfo.DEFAULT_GROUP; + } + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo == null) { + throw new ConstraintException( + "Default RSGroup (" + groupName + ") for this table's namespace does not exist."); + } + if (!rsGroupInfo.containsTable(desc.getTableName())) { + LOG.debug("Pre-moving table " + desc.getTableName() + " to RSGroup " + groupName); + groupAdminServer.moveTables(Sets.newHashSet(desc.getTableName()), groupName); + } + } + + ///////////////////////////////////////////////////////////////////////////// + // MasterObserver overrides + ///////////////////////////////////////////////////////////////////////////// + + private boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { + String groupName; + try { + groupName = master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) + .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (groupName == null) { + groupName = RSGroupInfo.DEFAULT_GROUP; + } + } catch (MasterNotRunningException | PleaseHoldException e) { + LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + + RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } + + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo == null) { + throw new ConstraintException( + "Default RSGroup (" + groupName + ") for this table's " + "namespace does not exist."); + } + + for (ServerName onlineServer : master.getServerManager().createDestinationServersList()) { + if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) { + return true; + } + } + return false; + } + + @Override + public void preCreateTableAction(final ObserverContext ctx, + final TableDescriptor desc, final RegionInfo[] regions) throws IOException { + if (!desc.getTableName().isSystemTable() && !rsgroupHasServersOnline(desc)) { + throw new HBaseIOException("No online servers in the rsgroup, which table " + + desc.getTableName().getNameAsString() + " belongs to"); + } + } + + // Assign table to default RSGroup. + @Override + public void postCreateTable(ObserverContext ctx, + TableDescriptor desc, RegionInfo[] regions) throws IOException { + assignTableToGroup(desc); + } + + // Remove table from its RSGroup. + @Override + public void postDeleteTable(ObserverContext ctx, + TableName tableName) throws IOException { + try { + RSGroupInfo group = groupAdminServer.getRSGroupInfoOfTable(tableName); + if (group != null) { + LOG.debug(String.format("Removing deleted table '%s' from rsgroup '%s'", tableName, + group.getName())); + groupAdminServer.moveTables(Sets.newHashSet(tableName), null); + } + } catch (IOException ex) { + LOG.debug("Failed to perform RSGroup information cleanup for table: " + tableName, ex); + } + } + + @Override + public void preCreateNamespace(ObserverContext ctx, + NamespaceDescriptor ns) throws IOException { + String group = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (group != null && groupAdminServer.getRSGroupInfo(group) == null) { + throw new ConstraintException("Region server group " + group + " does not exit"); + } + } + + @Override + public void preModifyNamespace(ObserverContext ctx, + NamespaceDescriptor currentNsDesc, NamespaceDescriptor newNsDesc) throws IOException { + preCreateNamespace(ctx, newNsDesc); + } + + @Override + public void preCloneSnapshot(ObserverContext ctx, + SnapshotDescription snapshot, TableDescriptor desc) throws IOException { + assignTableToGroup(desc); + } + + @Override + public void postClearDeadServers(ObserverContext ctx, + List servers, List notClearedServers) throws IOException { + Set
clearedServer = + servers.stream().filter(server -> !notClearedServers.contains(server)) + .map(ServerName::getAddress).collect(Collectors.toSet()); + if (!clearedServer.isEmpty()) { + groupAdminServer.removeServers(clearedServer); + } + } +} diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java new file mode 100644 index 000000000000..918a4fead8a1 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java @@ -0,0 +1,378 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import java.io.IOException; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersResponse; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.AccessChecker; +import org.apache.hadoop.hbase.security.access.Permission.Action; + +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; + +/** + * Implementation of RSGroupAdminService defined in RSGroupAdmin.proto. This class calls + * {@link RSGroupAdminServer} for actual work, converts result to protocol buffer response, handles + * exceptions if any occurred and then calls the {@code RpcCallback} with the response. + */ +class RSGroupAdminServiceImpl extends RSGroupAdminProtos.RSGroupAdminService { + + private MasterServices master; + + private RSGroupAdminServer groupAdminServer; + + private AccessChecker accessChecker; + + /** Provider for mapping principal names to Users */ + private UserProvider userProvider; + + RSGroupAdminServiceImpl() { + } + + void initialize(MasterServices master, RSGroupAdminServer groupAdminServer, + AccessChecker accessChecker, UserProvider userProvider) { + this.master = master; + this.groupAdminServer = groupAdminServer; + this.accessChecker = accessChecker; + this.userProvider = userProvider; + } + + @VisibleForTesting + void checkPermission(String request) throws IOException { + accessChecker.requirePermission(getActiveUser(), request, null, Action.ADMIN); + } + + /** + * Returns the active user to which authorization checks should be applied. If we are in the + * context of an RPC call, the remote user is used, otherwise the currently logged in user is + * used. + */ + private User getActiveUser() throws IOException { + // for non-rpc handling, fallback to system user + Optional optionalUser = RpcServer.getRequestUser(); + if (optionalUser.isPresent()) { + return optionalUser.get(); + } + return userProvider.getCurrent(); + } + + @Override + public void getRSGroupInfo(RpcController controller, GetRSGroupInfoRequest request, + RpcCallback done) { + GetRSGroupInfoResponse.Builder builder = GetRSGroupInfoResponse.newBuilder(); + String groupName = request.getRSGroupName(); + RSGroupAdminEndpoint.LOG.info( + master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, group=" + groupName); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preGetRSGroupInfo(groupName); + } + checkPermission("getRSGroupInfo"); + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(rsGroupInfo)); + } + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postGetRSGroupInfo(groupName); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void getRSGroupInfoOfTable(RpcController controller, GetRSGroupInfoOfTableRequest request, + RpcCallback done) { + GetRSGroupInfoOfTableResponse.Builder builder = GetRSGroupInfoOfTableResponse.newBuilder(); + TableName tableName = ProtobufUtil.toTableName(request.getTableName()); + RSGroupAdminEndpoint.LOG.info( + master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, table=" + tableName); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preGetRSGroupInfoOfTable(tableName); + } + checkPermission("getRSGroupInfoOfTable"); + RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupInfoOfTable(tableName); + if (RSGroupInfo != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo)); + } + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postGetRSGroupInfoOfTable(tableName); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveServers(RpcController controller, MoveServersRequest request, + RpcCallback done) { + MoveServersResponse.Builder builder = MoveServersResponse.newBuilder(); + Set
hostPorts = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); + } + RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts + + " to rsgroup " + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveServers(hostPorts, request.getTargetGroup()); + } + checkPermission("moveServers"); + groupAdminServer.moveServers(hostPorts, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveServers(hostPorts, request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveTables(RpcController controller, MoveTablesRequest request, + RpcCallback done) { + MoveTablesResponse.Builder builder = MoveTablesResponse.newBuilder(); + Set tables = new HashSet<>(request.getTableNameList().size()); + for (HBaseProtos.TableName tableName : request.getTableNameList()) { + tables.add(ProtobufUtil.toTableName(tableName)); + } + RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move tables " + tables + + " to rsgroup " + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveTables(tables, request.getTargetGroup()); + } + checkPermission("moveTables"); + groupAdminServer.moveTables(tables, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveTables(tables, request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void addRSGroup(RpcController controller, AddRSGroupRequest request, + RpcCallback done) { + AddRSGroupResponse.Builder builder = AddRSGroupResponse.newBuilder(); + RSGroupAdminEndpoint.LOG + .info(master.getClientIdAuditPrefix() + " add rsgroup " + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preAddRSGroup(request.getRSGroupName()); + } + checkPermission("addRSGroup"); + groupAdminServer.addRSGroup(request.getRSGroupName()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postAddRSGroup(request.getRSGroupName()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void removeRSGroup(RpcController controller, RemoveRSGroupRequest request, + RpcCallback done) { + RemoveRSGroupResponse.Builder builder = RemoveRSGroupResponse.newBuilder(); + RSGroupAdminEndpoint.LOG + .info(master.getClientIdAuditPrefix() + " remove rsgroup " + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preRemoveRSGroup(request.getRSGroupName()); + } + checkPermission("removeRSGroup"); + groupAdminServer.removeRSGroup(request.getRSGroupName()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postRemoveRSGroup(request.getRSGroupName()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void balanceRSGroup(RpcController controller, BalanceRSGroupRequest request, + RpcCallback done) { + BalanceRSGroupResponse.Builder builder = BalanceRSGroupResponse.newBuilder(); + RSGroupAdminEndpoint.LOG.info( + master.getClientIdAuditPrefix() + " balance rsgroup, group=" + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preBalanceRSGroup(request.getRSGroupName()); + } + checkPermission("balanceRSGroup"); + boolean balancerRan = groupAdminServer.balanceRSGroup(request.getRSGroupName()); + builder.setBalanceRan(balancerRan); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postBalanceRSGroup(request.getRSGroupName(), balancerRan); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + builder.setBalanceRan(false); + } + done.run(builder.build()); + } + + @Override + public void listRSGroupInfos(RpcController controller, ListRSGroupInfosRequest request, + RpcCallback done) { + ListRSGroupInfosResponse.Builder builder = ListRSGroupInfosResponse.newBuilder(); + RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " list rsgroup"); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preListRSGroups(); + } + checkPermission("listRSGroup"); + for (RSGroupInfo RSGroupInfo : groupAdminServer.listRSGroups()) { + builder.addRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo)); + } + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postListRSGroups(); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void getRSGroupInfoOfServer(RpcController controller, + GetRSGroupInfoOfServerRequest request, RpcCallback done) { + GetRSGroupInfoOfServerResponse.Builder builder = GetRSGroupInfoOfServerResponse.newBuilder(); + Address hp = + Address.fromParts(request.getServer().getHostName(), request.getServer().getPort()); + RSGroupAdminEndpoint.LOG + .info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, server=" + hp); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preGetRSGroupInfoOfServer(hp); + } + checkPermission("getRSGroupInfoOfServer"); + RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp); + if (info != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(info)); + } + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postGetRSGroupInfoOfServer(hp); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveServersAndTables(RpcController controller, MoveServersAndTablesRequest request, + RpcCallback done) { + MoveServersAndTablesResponse.Builder builder = MoveServersAndTablesResponse.newBuilder(); + Set
hostPorts = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); + } + Set tables = new HashSet<>(request.getTableNameList().size()); + for (HBaseProtos.TableName tableName : request.getTableNameList()) { + tables.add(ProtobufUtil.toTableName(tableName)); + } + RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts + + " and tables " + tables + " to rsgroup" + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveServersAndTables(hostPorts, tables, + request.getTargetGroup()); + } + checkPermission("moveServersAndTables"); + groupAdminServer.moveServersAndTables(hostPorts, tables, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveServersAndTables(hostPorts, tables, + request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void removeServers(RpcController controller, RemoveServersRequest request, + RpcCallback done) { + RemoveServersResponse.Builder builder = RemoveServersResponse.newBuilder(); + Set
servers = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + servers.add(Address.fromParts(el.getHostName(), el.getPort())); + } + RSGroupAdminEndpoint.LOG.info( + master.getClientIdAuditPrefix() + " remove decommissioned servers from rsgroup: " + servers); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preRemoveServers(servers); + } + checkPermission("removeServers"); + groupAdminServer.removeServers(servers); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postRemoveServers(servers); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } +} \ No newline at end of file diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java similarity index 92% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index 3f3e642e7598..a0af19233346 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.rsgroup; import java.io.IOException; @@ -56,18 +55,17 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Maps; /** - * GroupBasedLoadBalancer, used when Region Server Grouping is configured (HBase-6721) - * It does region balance based on a table's group membership. - * - * Most assignment methods contain two exclusive code paths: Online - when the group - * table is online and Offline - when it is unavailable. - * - * During Offline, assignments are assigned based on cached information in zookeeper. - * If unavailable (ie bootstrap) then regions are assigned randomly. - * - * Once the GROUP table has been assigned, the balancer switches to Online and will then - * start providing appropriate assignments for user tables. - * + * GroupBasedLoadBalancer, used when Region Server Grouping is configured (HBase-6721) It does + * region balance based on a table's group membership. + *

+ * Most assignment methods contain two exclusive code paths: Online - when the group table is online + * and Offline - when it is unavailable. + *

+ * During Offline, assignments are assigned based on cached information in zookeeper. If unavailable + * (ie bootstrap) then regions are assigned randomly. + *

+ * Once the GROUP table has been assigned, the balancer switches to Online and will then start + * providing appropriate assignments for user tables. */ @InterfaceAudience.Private public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { @@ -121,8 +119,8 @@ public List balanceCluster(TableName tableName, Map balanceCluster(Map> clusterState) throws HBaseIOException { if (!isOnline()) { - throw new ConstraintException(RSGroupInfoManager.RSGROUP_TABLE_NAME + - " is not online, unable to perform balance"); + throw new ConstraintException( + RSGroupInfoManager.class.getSimpleName() + " is not online, unable to perform balance"); } // Calculate correct assignments and a list of RegionPlan for mis-placed regions @@ -274,24 +272,20 @@ private void generateGroupMaps( List servers, ListMultimap regionMap, ListMultimap serverMap) throws HBaseIOException { - try { - for (RegionInfo region : regions) { - String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); - if (groupName == null) { - LOG.info("Group not found for table " + region.getTable() + ", using default"); - groupName = RSGroupInfo.DEFAULT_GROUP; - } - regionMap.put(groupName, region); + for (RegionInfo region : regions) { + String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; } - for (String groupKey : regionMap.keySet()) { - RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupKey); - serverMap.putAll(groupKey, filterOfflineServers(info, servers)); - if(serverMap.get(groupKey).size() < 1) { - serverMap.put(groupKey, LoadBalancer.BOGUS_SERVER_NAME); - } + regionMap.put(groupName, region); + } + for (String groupKey : regionMap.keySet()) { + RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupKey); + serverMap.putAll(groupKey, filterOfflineServers(info, servers)); + if (serverMap.get(groupKey).size() < 1) { + serverMap.put(groupKey, LoadBalancer.BOGUS_SERVER_NAME); } - } catch(IOException e) { - throw new HBaseIOException("Failed to generate group maps", e); } } @@ -376,17 +370,12 @@ private Pair>, List> correctAssignm List regions = assignments.getValue(); for (RegionInfo region : regions) { RSGroupInfo targetRSGInfo = null; - try { - String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); - if (groupName == null) { - LOG.info("Group not found for table " + region.getTable() + ", using default"); - groupName = RSGroupInfo.DEFAULT_GROUP; - } - targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName); - } catch (IOException exp) { - LOG.debug("RSGroup information null for region of table " + region.getTable(), - exp); + String groupName = rsGroupInfoManager.getRSGroupOfTable(region.getTable()); + if (groupName == null) { + LOG.info("Group not found for table " + region.getTable() + ", using default"); + groupName = RSGroupInfo.DEFAULT_GROUP; } + targetRSGInfo = rsGroupInfoManager.getRSGroup(groupName); if (targetRSGInfo == null || !targetRSGInfo.containsServer(currentHostServer.getAddress())) { // region is mis-placed regionPlansForMisplacedRegions.add(new RegionPlan(region, currentHostServer, null)); diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java similarity index 88% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java index 754962687f96..211fefdc0793 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java @@ -15,10 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hbase.rsgroup; -import com.google.protobuf.ServiceException; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; @@ -36,10 +34,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.AsyncClusterConnection; +import org.apache.hadoop.hbase.client.AsyncTable; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; -import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.CoprocessorDescriptorBuilder; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -48,14 +48,11 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.constraint.ConstraintException; import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.ServerListener; import org.apache.hadoop.hbase.master.TableStateManager; @@ -66,10 +63,14 @@ import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.protobuf.ProtobufMagic; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos; +import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsResponse; import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; @@ -79,6 +80,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Maps; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; @@ -91,21 +93,39 @@ * RSGroupInfo Map at {@link #rsGroupMap} and a Map of tables to the name of the rsgroup they belong * too (in {@link #tableMap}). These Maps are persisted to the hbase:rsgroup table (and cached in * zk) on each modification. - *

+ *

* Mutations on state are synchronized but reads can continue without having to wait on an instance * monitor, mutations do wholesale replace of the Maps on update -- Copy-On-Write; the local Maps of * state are read-only, just-in-case (see flushConfig). - *

+ *

* Reads must not block else there is a danger we'll deadlock. - *

+ *

* Clients of this class, the {@link RSGroupAdminEndpoint} for example, want to query and then act * on the results of the query modifying cache in zookeeper without another thread making * intermediate modifications. These clients synchronize on the 'this' instance so no other has * access concurrently. Reads must be able to continue concurrently. */ @InterfaceAudience.Private -final class RSGroupInfoManagerImpl implements RSGroupInfoManager { - private static final Logger LOG = LoggerFactory.getLogger(RSGroupInfoManagerImpl.class); +public class RSGroupInfoManager { + private static final Logger LOG = LoggerFactory.getLogger(RSGroupInfoManager.class); + + private static final String REASSIGN_WAIT_INTERVAL_KEY = "hbase.rsgroup.reassign.wait"; + private static final long DEFAULT_REASSIGN_WAIT_INTERVAL = 30 * 1000L; + + // Assigned before user tables + @VisibleForTesting + static final TableName RSGROUP_TABLE_NAME = + TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "rsgroup"); + + private static final String RS_GROUP_ZNODE = "rsgroup"; + + @VisibleForTesting + static final byte[] META_FAMILY_BYTES = Bytes.toBytes("m"); + + @VisibleForTesting + static final byte[] META_QUALIFIER_BYTES = Bytes.toBytes("i"); + + private static final byte[] ROW_KEY = { 0 }; /** Table descriptor for hbase:rsgroup catalog table */ private static final TableDescriptor RSGROUP_TABLE_DESC; @@ -129,7 +149,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { private volatile Map tableMap = Collections.emptyMap(); private final MasterServices masterServices; - private final Connection conn; + private final AsyncClusterConnection conn; private final ZKWatcher watcher; private final RSGroupStartupWorker rsGroupStartupWorker; // contains list of groups that were last flushed to persistent store @@ -138,15 +158,14 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { new ServerEventsListenerThread(); private FailedOpenUpdaterThread failedOpenUpdaterThread; - private RSGroupInfoManagerImpl(MasterServices masterServices) throws IOException { + public RSGroupInfoManager(MasterServices masterServices) throws IOException { this.masterServices = masterServices; this.watcher = masterServices.getZooKeeper(); - this.conn = masterServices.getConnection(); + this.conn = masterServices.getAsyncClusterConnection(); this.rsGroupStartupWorker = new RSGroupStartupWorker(); } - - private synchronized void init() throws IOException { + public synchronized void init() throws IOException { refresh(); serverEventsListenerThread.start(); masterServices.getServerManager().registerListener(serverEventsListenerThread); @@ -155,18 +174,11 @@ private synchronized void init() throws IOException { masterServices.getServerManager().registerListener(failedOpenUpdaterThread); } - static RSGroupInfoManager getInstance(MasterServices master) throws IOException { - RSGroupInfoManagerImpl instance = new RSGroupInfoManagerImpl(master); - instance.init(); - return instance; - } - public void start() { // create system table of rsgroup rsGroupStartupWorker.start(); } - @Override public synchronized void addRSGroup(RSGroupInfo rsGroupInfo) throws IOException { checkGroupName(rsGroupInfo.getName()); if (rsGroupMap.get(rsGroupInfo.getName()) != null || @@ -202,7 +214,6 @@ private static Set

getOnlineServers(final MasterServices master) { return onlineServers; } - @Override public synchronized Set
moveServers(Set
servers, String srcGroup, String dstGroup) throws IOException { RSGroupInfo src = getRSGroupInfo(srcGroup); @@ -232,7 +243,6 @@ public synchronized Set
moveServers(Set
servers, String srcGro return dst.getServers(); } - @Override public RSGroupInfo getRSGroupOfServer(Address serverHostPort) throws IOException { for (RSGroupInfo info : rsGroupMap.values()) { if (info.containsServer(serverHostPort)) { @@ -242,17 +252,14 @@ public RSGroupInfo getRSGroupOfServer(Address serverHostPort) throws IOException return null; } - @Override public RSGroupInfo getRSGroup(String groupName) { return rsGroupMap.get(groupName); } - @Override public String getRSGroupOfTable(TableName tableName) { return tableMap.get(tableName); } - @Override public synchronized void moveTables(Set tableNames, String groupName) throws IOException { // Check if rsGroupMap contains the destination rsgroup @@ -285,7 +292,6 @@ public synchronized void moveTables(Set tableNames, String groupName) flushConfig(newGroupMap); } - @Override public synchronized void removeRSGroup(String groupName) throws IOException { if (!rsGroupMap.containsKey(groupName) || groupName.equals(RSGroupInfo.DEFAULT_GROUP)) { throw new DoNotRetryIOException( @@ -296,17 +302,14 @@ public synchronized void removeRSGroup(String groupName) throws IOException { flushConfig(newGroupMap); } - @Override public List listRSGroups() { return Lists.newLinkedList(rsGroupMap.values()); } - @Override public boolean isOnline() { return rsGroupStartupWorker.isOnline(); } - @Override public void moveServersAndTables(Set
servers, Set tables, String srcGroup, String dstGroup) throws IOException { // get server's group @@ -331,7 +334,6 @@ public void moveServersAndTables(Set
servers, Set tables, St flushConfig(newGroupMap); } - @Override public synchronized void removeServers(Set
servers) throws IOException { Map rsGroupInfos = new HashMap(); for (Address el : servers) { @@ -357,25 +359,25 @@ public synchronized void removeServers(Set
servers) throws IOException } } - List retrieveGroupListFromGroupTable() throws IOException { + private List retrieveGroupListFromGroupTable() throws IOException { List rsGroupInfoList = Lists.newArrayList(); - try (Table table = conn.getTable(RSGROUP_TABLE_NAME); - ResultScanner scanner = table.getScanner(new Scan())) { + AsyncTable table = conn.getTable(RSGROUP_TABLE_NAME); + try (ResultScanner scanner = table.getScanner(META_FAMILY_BYTES, META_QUALIFIER_BYTES)) { for (Result result;;) { result = scanner.next(); if (result == null) { break; } RSGroupProtos.RSGroupInfo proto = RSGroupProtos.RSGroupInfo - .parseFrom(result.getValue(META_FAMILY_BYTES, META_QUALIFIER_BYTES)); - rsGroupInfoList.add(RSGroupProtobufUtil.toGroupInfo(proto)); + .parseFrom(result.getValue(META_FAMILY_BYTES, META_QUALIFIER_BYTES)); + rsGroupInfoList.add(ProtobufUtil.toGroupInfo(proto)); } } return rsGroupInfoList; } - List retrieveGroupListFromZookeeper() throws IOException { - String groupBasePath = ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, rsGroupZNode); + private List retrieveGroupListFromZookeeper() throws IOException { + String groupBasePath = ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE); List RSGroupInfoList = Lists.newArrayList(); // Overwrite any info stored by table, this takes precedence try { @@ -391,7 +393,7 @@ List retrieveGroupListFromZookeeper() throws IOException { ByteArrayInputStream bis = new ByteArrayInputStream(data, ProtobufUtil.lengthOfPBMagic(), data.length); RSGroupInfoList - .add(RSGroupProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis))); + .add(ProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis))); } } LOG.debug("Read ZK GroupInfo count:" + RSGroupInfoList.size()); @@ -402,7 +404,6 @@ List retrieveGroupListFromZookeeper() throws IOException { return RSGroupInfoList; } - @Override public void refresh() throws IOException { refresh(false); } @@ -467,7 +468,7 @@ private synchronized Map flushConfigTable(Map newGroupMap) thro resetRSGroupAndTableMaps(newGroupMap, newTableMap); try { - String groupBasePath = ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, rsGroupZNode); + String groupBasePath = + ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE); ZKUtil.createAndFailSilent(watcher, groupBasePath, ProtobufMagic.PB_MAGIC); List zkOps = new ArrayList<>(newGroupMap.size()); @@ -540,7 +542,7 @@ private synchronized void flushConfig(Map newGroupMap) thro for (RSGroupInfo RSGroupInfo : newGroupMap.values()) { String znode = ZNodePaths.joinZNode(groupBasePath, RSGroupInfo.getName()); - RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo); + RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo); LOG.debug("Updating znode: " + znode); ZKUtil.createAndFailSilent(watcher, znode); zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode)); @@ -646,7 +648,7 @@ private void updateFailedAssignments() { } /** - * Calls {@link RSGroupInfoManagerImpl#updateDefaultServers(SortedSet)} to update list of known + * Calls {@link RSGroupInfoManager#updateDefaultServers(SortedSet)} to update list of known * servers. Notifications about server changes are received by registering {@link ServerListener}. * As a listener, we need to return immediately, so the real work of updating the servers is done * asynchronously in this thread. @@ -681,9 +683,9 @@ public void run() { while (isMasterRunning(masterServices)) { try { LOG.info("Updating default servers."); - SortedSet
servers = RSGroupInfoManagerImpl.this.getDefaultServers(); + SortedSet
servers = RSGroupInfoManager.this.getDefaultServers(); if (!servers.equals(prevDefaultServers)) { - RSGroupInfoManagerImpl.this.updateDefaultServers(servers); + RSGroupInfoManager.this.updateDefaultServers(servers); prevDefaultServers = servers; LOG.info("Updated with servers: " + servers.size()); } @@ -790,15 +792,12 @@ private boolean waitForGroupTableOnline() { createRSGroupTable(); } // try reading from the table - try (Table table = conn.getTable(RSGROUP_TABLE_NAME)) { - table.get(new Get(ROW_KEY)); - } - LOG.info( - "RSGroup table=" + RSGROUP_TABLE_NAME + " is online, refreshing cached information"); - RSGroupInfoManagerImpl.this.refresh(true); + FutureUtils.get(conn.getTable(RSGROUP_TABLE_NAME).get(new Get(ROW_KEY))); + LOG.info("RSGroup table={} is online, refreshing cached information", RSGROUP_TABLE_NAME); + RSGroupInfoManager.this.refresh(true); online = true; // flush any inconsistencies between ZK and HTable - RSGroupInfoManagerImpl.this.flushConfig(); + RSGroupInfoManager.this.flushConfig(); return true; } catch (Exception e) { LOG.warn("Failed to perform check", e); @@ -836,8 +835,8 @@ private void createRSGroupTable() throws IOException { } else { Procedure result = masterServices.getMasterProcedureExecutor().getResult(procId); if (result != null && result.isFailed()) { - throw new IOException( - "Failed to create group table. " + MasterProcedureUtil.unwrapRemoteIOException(result)); + throw new IOException("Failed to create group table. " + + MasterProcedureUtil.unwrapRemoteIOException(result)); } } } @@ -852,33 +851,24 @@ private static boolean isMasterRunning(MasterServices masterServices) { } private void multiMutate(List mutations) throws IOException { - try (Table table = conn.getTable(RSGROUP_TABLE_NAME)) { - CoprocessorRpcChannel channel = table.coprocessorService(ROW_KEY); - MultiRowMutationProtos.MutateRowsRequest.Builder mmrBuilder = - MultiRowMutationProtos.MutateRowsRequest.newBuilder(); - for (Mutation mutation : mutations) { - if (mutation instanceof Put) { - mmrBuilder.addMutationRequest(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toMutation( - org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType.PUT, - mutation)); - } else if (mutation instanceof Delete) { - mmrBuilder.addMutationRequest(org.apache.hadoop.hbase.protobuf.ProtobufUtil.toMutation( - org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType.DELETE, - mutation)); - } else { - throw new DoNotRetryIOException( + MutateRowsRequest.Builder builder = MutateRowsRequest.newBuilder(); + for (Mutation mutation : mutations) { + if (mutation instanceof Put) { + builder + .addMutationRequest(ProtobufUtil.toMutation(MutationProto.MutationType.PUT, mutation)); + } else if (mutation instanceof Delete) { + builder.addMutationRequest( + ProtobufUtil.toMutation(MutationProto.MutationType.DELETE, mutation)); + } else { + throw new DoNotRetryIOException( "multiMutate doesn't support " + mutation.getClass().getName()); - } - } - - MultiRowMutationProtos.MultiRowMutationService.BlockingInterface service = - MultiRowMutationProtos.MultiRowMutationService.newBlockingStub(channel); - try { - service.mutateRows(null, mmrBuilder.build()); - } catch (ServiceException ex) { - ProtobufUtil.toIOException(ex); } } + MutateRowsRequest request = builder.build(); + AsyncTable table = conn.getTable(RSGROUP_TABLE_NAME); + FutureUtils.get(table. coprocessorService( + MultiRowMutationService::newStub, + (stub, controller, done) -> stub.mutateRows(controller, request, done), ROW_KEY)); } private void checkGroupName(String groupName) throws ConstraintException { @@ -886,4 +876,4 @@ private void checkGroupName(String groupName) throws ConstraintException { throw new ConstraintException("RSGroup name should only contain alphanumeric characters"); } } -} +} \ No newline at end of file diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java similarity index 100% rename from hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java rename to hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java index cbfdd3f74497..2b22116c0436 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.SyncReplicationState; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager; import org.apache.hadoop.hbase.security.access.AccessChecker; import org.apache.hadoop.hbase.security.access.ZKPermissionWatcher; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; @@ -486,6 +487,11 @@ public List executeRegionPlansWithThrottling(List plans) return null; } + @Override + public RSGroupInfoManager getRSRSGroupInfoManager() { + return null; + } + @Override public AsyncClusterConnection getAsyncClusterConnection() { return null; diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java similarity index 100% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java similarity index 81% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java index 0278e3cfbfa1..c1c157a303e6 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.util.Bytes; - import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -49,11 +48,11 @@ import org.slf4j.LoggerFactory; /** - * Performs authorization checks for rsgroup operations, according to different - * levels of authorized users. + * Performs authorization checks for rsgroup operations, according to different levels of authorized + * users. */ -@Category({SecurityTests.class, MediumTests.class}) -public class TestRSGroupsWithACL extends SecureTestUtil{ +@Category({ SecurityTests.class, MediumTests.class }) +public class TestRSGroupsWithACL extends SecureTestUtil { @ClassRule public static final HBaseClassTestRule CLASS_RULE = @@ -98,8 +97,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil{ public static void setupBeforeClass() throws Exception { // setup configuration conf = TEST_UTIL.getConfiguration(); - conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, - RSGroupBasedLoadBalancer.class.getName()); + conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); // Enable security enableSecurity(conf); // Verify enableSecurity sets up what we require @@ -108,8 +106,8 @@ public static void setupBeforeClass() throws Exception { configureRSGroupAdminEndpoint(conf); TEST_UTIL.startMiniCluster(); - rsGroupAdminEndpoint = (RSGroupAdminEndpoint) TEST_UTIL.getMiniHBaseCluster().getMaster(). - getMasterCoprocessorHost().findCoprocessor(RSGroupAdminEndpoint.class.getName()); + rsGroupAdminEndpoint = (RSGroupAdminEndpoint) TEST_UTIL.getMiniHBaseCluster().getMaster() + .getMasterCoprocessorHost().findCoprocessor(RSGroupAdminEndpoint.class.getName()); // Wait for the ACL table to become available TEST_UTIL.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); @@ -141,31 +139,21 @@ private static void setUpTableAndUserPermissions() throws Exception { cfd.setMaxVersions(100); tableBuilder.setColumnFamily(cfd.build()); tableBuilder.setValue(TableDescriptorBuilder.OWNER, USER_OWNER.getShortName()); - createTable(TEST_UTIL, tableBuilder.build(), - new byte[][] { Bytes.toBytes("s") }); + createTable(TEST_UTIL, tableBuilder.build(), new byte[][] { Bytes.toBytes("s") }); // Set up initial grants - grantGlobal(TEST_UTIL, USER_ADMIN.getShortName(), - Permission.Action.ADMIN, - Permission.Action.CREATE, - Permission.Action.READ, - Permission.Action.WRITE); + grantGlobal(TEST_UTIL, USER_ADMIN.getShortName(), Permission.Action.ADMIN, + Permission.Action.CREATE, Permission.Action.READ, Permission.Action.WRITE); - grantOnTable(TEST_UTIL, USER_RW.getShortName(), - TEST_TABLE, TEST_FAMILY, null, - Permission.Action.READ, - Permission.Action.WRITE); + grantOnTable(TEST_UTIL, USER_RW.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ, Permission.Action.WRITE); // USER_CREATE is USER_RW plus CREATE permissions - grantOnTable(TEST_UTIL, USER_CREATE.getShortName(), - TEST_TABLE, null, null, - Permission.Action.CREATE, - Permission.Action.READ, - Permission.Action.WRITE); + grantOnTable(TEST_UTIL, USER_CREATE.getShortName(), TEST_TABLE, null, null, + Permission.Action.CREATE, Permission.Action.READ, Permission.Action.WRITE); - grantOnTable(TEST_UTIL, USER_RO.getShortName(), - TEST_TABLE, TEST_FAMILY, null, - Permission.Action.READ); + grantOnTable(TEST_UTIL, USER_RO.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ); grantGlobal(TEST_UTIL, toGroupEntry(GROUP_ADMIN), Permission.Action.ADMIN); grantGlobal(TEST_UTIL, toGroupEntry(GROUP_CREATE), Permission.Action.CREATE); @@ -174,8 +162,8 @@ private static void setUpTableAndUserPermissions() throws Exception { assertEquals(4, PermissionStorage.getTablePermissions(conf, TEST_TABLE).size()); try { - assertEquals(4, AccessControlClient.getUserPermissions(systemUserConnection, - TEST_TABLE.toString()).size()); + assertEquals(4, + AccessControlClient.getUserPermissions(systemUserConnection, TEST_TABLE.toString()).size()); } catch (AssertionError e) { fail(e.getMessage()); } catch (Throwable e) { @@ -210,14 +198,13 @@ private static void configureRSGroupAdminEndpoint(Configuration conf) { coprocessors += "," + currentCoprocessors; } conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, coprocessors); - conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, - RSGroupBasedLoadBalancer.class.getName()); + conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); } @Test public void testGetRSGroupInfo() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("getRSGroupInfo"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfo"); return null; }; @@ -227,7 +214,7 @@ public void testGetRSGroupInfo() throws Exception { @Test public void testGetRSGroupInfoOfTable() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("getRSGroupInfoOfTable"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfoOfTable"); return null; }; @@ -237,7 +224,7 @@ public void testGetRSGroupInfoOfTable() throws Exception { @Test public void testMoveServers() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("moveServers"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveServers"); return null; }; @@ -247,7 +234,7 @@ public void testMoveServers() throws Exception { @Test public void testMoveTables() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("moveTables"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveTables"); return null; }; @@ -257,7 +244,7 @@ public void testMoveTables() throws Exception { @Test public void testAddRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("addRSGroup"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("addRSGroup"); return null; }; @@ -267,7 +254,7 @@ public void testAddRSGroup() throws Exception { @Test public void testRemoveRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("removeRSGroup"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("removeRSGroup"); return null; }; @@ -277,7 +264,7 @@ public void testRemoveRSGroup() throws Exception { @Test public void testBalanceRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("balanceRSGroup"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("balanceRSGroup"); return null; }; @@ -287,7 +274,7 @@ public void testBalanceRSGroup() throws Exception { @Test public void testListRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("listRSGroup"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("listRSGroup"); return null; }; @@ -297,7 +284,7 @@ public void testListRSGroup() throws Exception { @Test public void testGetRSGroupInfoOfServer() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("getRSGroupInfoOfServer"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfoOfServer"); return null; }; @@ -307,7 +294,7 @@ public void testGetRSGroupInfoOfServer() throws Exception { @Test public void testMoveServersAndTables() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("moveServersAndTables"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveServersAndTables"); return null; }; @@ -317,7 +304,7 @@ public void testMoveServersAndTables() throws Exception { @Test public void testRemoveServers() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.checkPermission("removeServers"); + rsGroupAdminEndpoint.getGroupAdminService().checkPermission("removeServers"); return null; }; @@ -326,7 +313,7 @@ public void testRemoveServers() throws Exception { private void validateAdminPermissions(AccessTestAction action) throws Exception { verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, - USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, + USER_GROUP_WRITE, USER_GROUP_CREATE); } } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java similarity index 95% rename from hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java index 88a43396d7b6..24ecb608631e 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -53,7 +52,7 @@ public VerifyingRSGroupAdminClient(RSGroupAdmin RSGroupAdmin, Configuration conf throws IOException { wrapped = RSGroupAdmin; table = ConnectionFactory.createConnection(conf) - .getTable(RSGroupInfoManager.RSGROUP_TABLE_NAME); + .getTable(RSGroupInfoManager.RSGROUP_TABLE_NAME); zkw = new ZKWatcher(conf, this.getClass().getSimpleName(), null); } @@ -129,7 +128,7 @@ public void verify() throws IOException { result.getValue( RSGroupInfoManager.META_FAMILY_BYTES, RSGroupInfoManager.META_QUALIFIER_BYTES)); - groupMap.put(proto.getName(), RSGroupProtobufUtil.toGroupInfo(proto)); + groupMap.put(proto.getName(), ProtobufUtil.toGroupInfo(proto)); } Assert.assertEquals(Sets.newHashSet(groupMap.values()), Sets.newHashSet(wrapped.listRSGroups())); @@ -141,7 +140,7 @@ public void verify() throws IOException { ProtobufUtil.expectPBMagicPrefix(data); ByteArrayInputStream bis = new ByteArrayInputStream( data, ProtobufUtil.lengthOfPBMagic(), data.length); - zList.add(RSGroupProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis))); + zList.add(ProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis))); } } Assert.assertEquals(zList.size(), groupMap.size()); diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index 7483da5968e0..a6b62ad5895d 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -187,41 +187,6 @@ - - rsgroup - - - !skip-rsgroup - - - - - org.apache.hbase - hbase-rsgroup - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-test-source - - add-test-source - - - - src/test/rsgroup - - - - - - - - skipShellTests diff --git a/pom.xml b/pom.xml index 65f3466fe002..f14d0a0a4196 100755 --- a/pom.xml +++ b/pom.xml @@ -1519,7 +1519,6 @@ hbase-procedure-${project.version}-tests.jar hbase-it-${project.version}-tests.jar hbase-annotations-${project.version}-tests.jar - hbase-rsgroup-${project.version}-tests.jar hbase-mapreduce-${project.version}-tests.jar hbase-zookeeper-${project.version}-tests.jar bash @@ -1647,18 +1646,6 @@ test-jar test - - hbase-rsgroup - org.apache.hbase - ${project.version} - - - hbase-rsgroup - org.apache.hbase - ${project.version} - test-jar - test - hbase-replication org.apache.hbase @@ -2286,17 +2273,6 @@ --> - - rsgroup - - - !skip-rsgroup - - - - hbase-rsgroup - - build-with-jdk8