Skip to content

Commit f278a4c

Browse files
Apache9Apache9
authored
HBASE-26587 Introduce a new Admin API to change SFT implementation (#4030)
Signed-off-by: Wellington Ramos Chevreuil <[email protected]> Reviewed-by: Josh Elser <[email protected]>
1 parent 621a1cd commit f278a4c

File tree

29 files changed

+1601
-252
lines changed

29 files changed

+1601
-252
lines changed

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,6 @@
6969
import org.apache.yetus.audience.InterfaceAudience;
7070

7171
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
72-
import org.apache.yetus.audience.InterfaceStability;
7372

7473
/**
7574
* The administrative API for HBase. Obtain an instance from {@link Connection#getAdmin()} and
@@ -500,6 +499,31 @@ default void modifyColumnFamily(TableName tableName, ColumnFamilyDescriptor colu
500499
Future<Void> modifyColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamily)
501500
throws IOException;
502501

502+
/**
503+
* Change the store file tracker of the given table's given family.
504+
* @param tableName the table you want to change
505+
* @param family the family you want to change
506+
* @param dstSFT the destination store file tracker
507+
* @throws IOException if a remote or network exception occurs
508+
*/
509+
default void modifyColumnFamilyStoreFileTracker(TableName tableName, byte[] family, String dstSFT)
510+
throws IOException {
511+
get(modifyColumnFamilyStoreFileTrackerAsync(tableName, family, dstSFT), getSyncWaitTimeout(),
512+
TimeUnit.MILLISECONDS);
513+
}
514+
515+
/**
516+
* Change the store file tracker of the given table's given family.
517+
* @param tableName the table you want to change
518+
* @param family the family you want to change
519+
* @param dstSFT the destination store file tracker
520+
* @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the
521+
* operation to complete
522+
* @throws IOException if a remote or network exception occurs
523+
*/
524+
Future<Void> modifyColumnFamilyStoreFileTrackerAsync(TableName tableName, byte[] family,
525+
String dstSFT) throws IOException;
526+
503527
/**
504528
* Get all the online regions on a region server.
505529
*
@@ -1058,6 +1082,28 @@ default void modifyTable(TableDescriptor td) throws IOException {
10581082
*/
10591083
Future<Void> modifyTableAsync(TableDescriptor td) throws IOException;
10601084

1085+
/**
1086+
* Change the store file tracker of the given table.
1087+
* @param tableName the table you want to change
1088+
* @param dstSFT the destination store file tracker
1089+
* @throws IOException if a remote or network exception occurs
1090+
*/
1091+
default void modifyTableStoreFileTracker(TableName tableName, String dstSFT) throws IOException {
1092+
get(modifyTableStoreFileTrackerAsync(tableName, dstSFT), getSyncWaitTimeout(),
1093+
TimeUnit.MILLISECONDS);
1094+
}
1095+
1096+
/**
1097+
* Change the store file tracker of the given table.
1098+
* @param tableName the table you want to change
1099+
* @param dstSFT the destination store file tracker
1100+
* @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the
1101+
* operation to complete
1102+
* @throws IOException if a remote or network exception occurs
1103+
*/
1104+
Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT)
1105+
throws IOException;
1106+
10611107
/**
10621108
* Shuts down the HBase cluster.
10631109
* <p/>

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,10 +230,16 @@ public Future<Void> deleteColumnFamilyAsync(TableName tableName, byte[] columnFa
230230

231231
@Override
232232
public Future<Void> modifyColumnFamilyAsync(TableName tableName,
233-
ColumnFamilyDescriptor columnFamily) throws IOException {
233+
ColumnFamilyDescriptor columnFamily) throws IOException {
234234
return admin.modifyColumnFamily(tableName, columnFamily);
235235
}
236236

237+
@Override
238+
public Future<Void> modifyColumnFamilyStoreFileTrackerAsync(TableName tableName, byte[] family,
239+
String dstSFT) throws IOException {
240+
return admin.modifyColumnFamilyStoreFileTracker(tableName, family, dstSFT);
241+
}
242+
237243
@Override
238244
public List<RegionInfo> getRegions(ServerName serverName) throws IOException {
239245
return get(admin.getRegions(serverName));
@@ -475,6 +481,12 @@ public Future<Void> modifyTableAsync(TableDescriptor td) throws IOException {
475481
return admin.modifyTable(td);
476482
}
477483

484+
@Override
485+
public Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT)
486+
throws IOException {
487+
return admin.modifyTableStoreFileTracker(tableName, dstSFT);
488+
}
489+
478490
@Override
479491
public void shutdown() throws IOException {
480492
get(admin.shutdown());

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,13 @@ CompletableFuture<Void> createTable(TableDescriptor desc, byte[] startKey, byte[
183183
*/
184184
CompletableFuture<Void> modifyTable(TableDescriptor desc);
185185

186+
/**
187+
* Change the store file tracker of the given table.
188+
* @param tableName the table you want to change
189+
* @param dstSFT the destination store file tracker
190+
*/
191+
CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT);
192+
186193
/**
187194
* Deletes a table.
188195
* @param tableName name of table to delete
@@ -252,6 +259,15 @@ CompletableFuture<Void> addColumnFamily(TableName tableName,
252259
CompletableFuture<Void> modifyColumnFamily(TableName tableName,
253260
ColumnFamilyDescriptor columnFamily);
254261

262+
/**
263+
* Change the store file tracker of the given table's given family.
264+
* @param tableName the table you want to change
265+
* @param family the family you want to change
266+
* @param dstSFT the destination store file tracker
267+
*/
268+
CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName, byte[] family,
269+
String dstSFT);
270+
255271
/**
256272
* Create a new namespace.
257273
* @param descriptor descriptor which describes the new namespace

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,11 @@ public CompletableFuture<Void> modifyTable(TableDescriptor desc) {
146146
return wrap(rawAdmin.modifyTable(desc));
147147
}
148148

149+
@Override
150+
public CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT) {
151+
return wrap(rawAdmin.modifyTableStoreFileTracker(tableName, dstSFT));
152+
}
153+
149154
@Override
150155
public CompletableFuture<Void> deleteTable(TableName tableName) {
151156
return wrap(rawAdmin.deleteTable(tableName));
@@ -198,6 +203,12 @@ public CompletableFuture<Void> modifyColumnFamily(TableName tableName,
198203
return wrap(rawAdmin.modifyColumnFamily(tableName, columnFamily));
199204
}
200205

206+
@Override
207+
public CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName,
208+
byte[] family, String dstSFT) {
209+
return wrap(rawAdmin.modifyColumnFamilyStoreFileTracker(tableName, family, dstSFT));
210+
}
211+
201212
@Override
202213
public CompletableFuture<Void> createNamespace(NamespaceDescriptor descriptor) {
203214
return wrap(rawAdmin.createNamespace(descriptor));

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
2222
import static org.apache.hadoop.hbase.util.FutureUtils.addListener;
2323
import static org.apache.hadoop.hbase.util.FutureUtils.unwrapCompletionException;
24+
2425
import edu.umd.cs.findbugs.annotations.Nullable;
2526
import java.io.IOException;
2627
import java.util.ArrayList;
@@ -230,10 +231,14 @@
230231
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsResponse;
231232
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
232233
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnResponse;
234+
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
235+
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerResponse;
233236
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
234237
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
235238
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
236239
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableResponse;
240+
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
241+
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse;
237242
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
238243
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionResponse;
239244
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
@@ -660,6 +665,18 @@ public CompletableFuture<Void> modifyTable(TableDescriptor desc) {
660665
(resp) -> resp.getProcId(), new ModifyTableProcedureBiConsumer(this, desc.getTableName()));
661666
}
662667

668+
@Override
669+
public CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT) {
670+
return this
671+
.<ModifyTableStoreFileTrackerRequest, ModifyTableStoreFileTrackerResponse> procedureCall(
672+
tableName,
673+
RequestConverter.buildModifyTableStoreFileTrackerRequest(tableName, dstSFT,
674+
ng.getNonceGroup(), ng.newNonce()),
675+
(s, c, req, done) -> s.modifyTableStoreFileTracker(c, req, done),
676+
(resp) -> resp.getProcId(),
677+
new ModifyTableStoreFileTrackerProcedureBiConsumer(this, tableName));
678+
}
679+
663680
@Override
664681
public CompletableFuture<Void> deleteTable(TableName tableName) {
665682
return this.<DeleteTableRequest, DeleteTableResponse> procedureCall(tableName,
@@ -808,6 +825,19 @@ public CompletableFuture<Void> modifyColumnFamily(TableName tableName,
808825
(resp) -> resp.getProcId(), new ModifyColumnFamilyProcedureBiConsumer(tableName));
809826
}
810827

828+
@Override
829+
public CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName,
830+
byte[] family, String dstSFT) {
831+
return this
832+
.<ModifyColumnStoreFileTrackerRequest, ModifyColumnStoreFileTrackerResponse> procedureCall(
833+
tableName,
834+
RequestConverter.buildModifyColumnStoreFileTrackerRequest(tableName, family, dstSFT,
835+
ng.getNonceGroup(), ng.newNonce()),
836+
(s, c, req, done) -> s.modifyColumnStoreFileTracker(c, req, done),
837+
(resp) -> resp.getProcId(),
838+
new ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer(tableName));
839+
}
840+
811841
@Override
812842
public CompletableFuture<Void> createNamespace(NamespaceDescriptor descriptor) {
813843
return this.<CreateNamespaceRequest, CreateNamespaceResponse> procedureCall(
@@ -2590,7 +2620,20 @@ private static class ModifyTableProcedureBiConsumer extends TableProcedureBiCons
25902620

25912621
@Override
25922622
String getOperationType() {
2593-
return "ENABLE";
2623+
return "MODIFY";
2624+
}
2625+
}
2626+
2627+
private static class ModifyTableStoreFileTrackerProcedureBiConsumer
2628+
extends TableProcedureBiConsumer {
2629+
2630+
ModifyTableStoreFileTrackerProcedureBiConsumer(AsyncAdmin admin, TableName tableName) {
2631+
super(tableName);
2632+
}
2633+
2634+
@Override
2635+
String getOperationType() {
2636+
return "MODIFY_TABLE_STORE_FILE_TRACKER";
25942637
}
25952638
}
25962639

@@ -2684,6 +2727,19 @@ String getOperationType() {
26842727
}
26852728
}
26862729

2730+
private static class ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer
2731+
extends TableProcedureBiConsumer {
2732+
2733+
ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer(TableName tableName) {
2734+
super(tableName);
2735+
}
2736+
2737+
@Override
2738+
String getOperationType() {
2739+
return "MODIFY_COLUMN_FAMILY_STORE_FILE_TRACKER";
2740+
}
2741+
}
2742+
26872743
private static class CreateNamespaceProcedureBiConsumer extends NamespaceProcedureBiConsumer {
26882744

26892745
CreateNamespaceProcedureBiConsumer(String namespaceName) {
@@ -4282,5 +4338,4 @@ public CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNam
42824338
return CompletableFuture.completedFuture(Collections.emptyList());
42834339
}
42844340
}
4285-
42864341
}

‎hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import org.apache.yetus.audience.InterfaceAudience;
7070

7171
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
72+
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
7273
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
7374
import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils;
7475

@@ -128,8 +129,10 @@
128129
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest;
129130
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest;
130131
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
132+
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
131133
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
132134
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
135+
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
133136
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
134137
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
135138
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineRegionRequest;
@@ -1022,6 +1025,19 @@ public static ModifyColumnRequest buildModifyColumnRequest(
10221025
return builder.build();
10231026
}
10241027

1028+
public static ModifyColumnStoreFileTrackerRequest
1029+
buildModifyColumnStoreFileTrackerRequest(final TableName tableName, final byte[] family,
1030+
final String dstSFT, final long nonceGroup, final long nonce) {
1031+
ModifyColumnStoreFileTrackerRequest.Builder builder =
1032+
ModifyColumnStoreFileTrackerRequest.newBuilder();
1033+
builder.setTableName(ProtobufUtil.toProtoTableName((tableName)));
1034+
builder.setFamily(ByteString.copyFrom(family));
1035+
builder.setDstSft(dstSFT);
1036+
builder.setNonceGroup(nonceGroup);
1037+
builder.setNonce(nonce);
1038+
return builder.build();
1039+
}
1040+
10251041
/**
10261042
* Create a protocol buffer MoveRegionRequest
10271043
* @param encodedRegionName
@@ -1220,6 +1236,17 @@ public static ModifyTableRequest buildModifyTableRequest(
12201236
return builder.build();
12211237
}
12221238

1239+
public static ModifyTableStoreFileTrackerRequest buildModifyTableStoreFileTrackerRequest(
1240+
final TableName tableName, final String dstSFT, final long nonceGroup, final long nonce) {
1241+
ModifyTableStoreFileTrackerRequest.Builder builder =
1242+
ModifyTableStoreFileTrackerRequest.newBuilder();
1243+
builder.setTableName(ProtobufUtil.toProtoTableName((tableName)));
1244+
builder.setDstSft(dstSFT);
1245+
builder.setNonceGroup(nonceGroup);
1246+
builder.setNonce(nonce);
1247+
return builder.build();
1248+
}
1249+
12231250
/**
12241251
* Creates a protocol buffer GetTableDescriptorsRequest
12251252
*

‎hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -732,6 +732,29 @@ message BalancerRejectionsResponse {
732732
repeated BalancerRejection balancer_rejection = 1;
733733
}
734734

735+
message ModifyTableStoreFileTrackerRequest {
736+
required TableName table_Name = 1;
737+
required string dst_sft = 2;
738+
optional uint64 nonce_group = 3 [default = 0];
739+
optional uint64 nonce = 4 [default = 0];
740+
}
741+
742+
message ModifyTableStoreFileTrackerResponse {
743+
optional uint64 proc_id = 1;
744+
}
745+
746+
message ModifyColumnStoreFileTrackerRequest {
747+
required TableName table_Name = 1;
748+
required bytes family = 2;
749+
required string dst_sft = 3;
750+
optional uint64 nonce_group = 4 [default = 0];
751+
optional uint64 nonce = 5 [default = 0];
752+
}
753+
754+
message ModifyColumnStoreFileTrackerResponse {
755+
optional uint64 proc_id = 1;
756+
}
757+
735758
service MasterService {
736759
/** Used by the client to get the number of regions that have received the updated schema */
737760
rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
@@ -1165,6 +1188,12 @@ service MasterService {
11651188

11661189
rpc GetLogEntries(LogRequest)
11671190
returns(LogEntry);
1191+
1192+
rpc ModifyTableStoreFileTracker(ModifyTableStoreFileTrackerRequest)
1193+
returns(ModifyTableStoreFileTrackerResponse);
1194+
1195+
rpc ModifyColumnStoreFileTracker(ModifyColumnStoreFileTrackerRequest)
1196+
returns(ModifyColumnStoreFileTrackerResponse);
11681197
}
11691198

11701199
// HBCK Service definitions.

‎hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,3 +658,18 @@ message ModifyTableDescriptorStateData {
658658
required TableName table_name = 1;
659659
optional TableSchema modified_table_schema = 2;
660660
}
661+
662+
enum ModifyStoreFileTrackerState {
663+
MODIFY_STORE_FILE_TRACKER_FINISH_PREVIOUS_MIGRATION = 1;
664+
MODIFY_STORE_FILE_TRACKER_START_MIGRATION = 2;
665+
MODIFY_STORE_FILE_TRACKER_FINISH_MIGRATION = 3;
666+
}
667+
668+
message ModifyStoreFileTrackerStateData {
669+
required TableName table_name = 1;
670+
required string dst_sft = 2;
671+
}
672+
673+
message ModifyColumnFamilyStoreFileTrackerStateData {
674+
required bytes family = 1;
675+
}

0 commit comments

Comments
 (0)