diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index feaa3a4ec7fb..2b50d76ca5d0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -2641,20 +2641,16 @@ public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName tableN ByteString encodedRegionName, Map> storeFiles, Map storeFilesSize, long bulkloadSeqId) { return toBulkLoadDescriptor(tableName, encodedRegionName, storeFiles, storeFilesSize, - bulkloadSeqId, null, true); + bulkloadSeqId, true); } public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName tableName, ByteString encodedRegionName, Map> storeFiles, - Map storeFilesSize, long bulkloadSeqId, List clusterIds, - boolean replicate) { + Map storeFilesSize, long bulkloadSeqId, boolean replicate) { BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder().setTableName(ProtobufUtil.toProtoTableName(tableName)) .setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId) .setReplicate(replicate); - if (clusterIds != null) { - desc.addAllClusterIds(clusterIds); - } for (Map.Entry> entry : storeFiles.entrySet()) { WALProtos.StoreDescriptor.Builder builder = diff --git a/hbase-protocol-shaded/src/main/protobuf/WAL.proto b/hbase-protocol-shaded/src/main/protobuf/WAL.proto index ba12dcf3edfc..9b5929ee9ca4 100644 --- a/hbase-protocol-shaded/src/main/protobuf/WAL.proto +++ b/hbase-protocol-shaded/src/main/protobuf/WAL.proto @@ -152,7 +152,6 @@ message BulkLoadDescriptor { required bytes encoded_region_name = 2; repeated StoreDescriptor stores = 3; required int64 bulkload_seq_num = 4; - repeated string cluster_ids = 5; optional bool replicate = 6 [default = true]; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 0113d439dc02..160b04efbeb6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -7077,8 +7077,11 @@ public Map> bulkLoadHFiles(Collection> f WALProtos.BulkLoadDescriptor loadDescriptor = ProtobufUtil.toBulkLoadDescriptor(this.getRegionInfo().getTable(), UnsafeByteOperations.unsafeWrap(this.getRegionInfo().getEncodedNameAsBytes()), - storeFiles, storeFilesSizes, seqId, clusterIds, replicate); + storeFiles, storeFilesSizes, seqId, replicate); WALUtil.writeBulkLoadMarkerAndSync(this.wal, this.getReplicationScope(), getRegionInfo(), + clusterIds == null + ? WALKey.EMPTY_UUIDS + : clusterIds.stream().map(UUID::fromString).collect(Collectors.toList()), loadDescriptor, mvcc); } catch (IOException ioe) { if (this.rsServices != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index e246da4bd83d..393699f838d0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -2418,11 +2418,6 @@ public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException { long start = EnvironmentEdgeManager.currentTime(); List clusterIds = new ArrayList<>(request.getClusterIdsList()); - if (clusterIds.contains(this.regionServer.clusterId)) { - return BulkLoadHFileResponse.newBuilder().setLoaded(true).build(); - } else { - clusterIds.add(this.regionServer.clusterId); - } try { checkOpen(); requestCount.increment(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java index dd8d152da8bb..215421a96d0e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALUtil.java @@ -18,12 +18,15 @@ package org.apache.hadoop.hbase.regionserver.wal; import static org.apache.hadoop.hbase.HConstants.REPLICATION_SCOPE_GLOBAL; +import static org.apache.hadoop.hbase.wal.WALKey.EMPTY_UUIDS; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; +import java.util.UUID; import java.util.function.Function; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -75,8 +78,8 @@ private WALUtil() { public static WALKeyImpl writeCompactionMarker(WAL wal, NavigableMap replicationScope, RegionInfo hri, final CompactionDescriptor c, MultiVersionConcurrencyControl mvcc) throws IOException { - WALKeyImpl walKey = - writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc, null); + WALKeyImpl walKey = writeMarker(wal, replicationScope, hri, EMPTY_UUIDS, + WALEdit.createCompaction(hri, c), mvcc, null); if (LOG.isTraceEnabled()) { LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c)); } @@ -91,7 +94,7 @@ public static WALKeyImpl writeCompactionMarker(WAL wal, public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap replicationScope, RegionInfo hri, final FlushDescriptor f, boolean sync, MultiVersionConcurrencyControl mvcc) throws IOException { - WALKeyImpl walKey = doFullMarkerAppendTransaction(wal, replicationScope, hri, + WALKeyImpl walKey = doFullMarkerAppendTransaction(wal, replicationScope, hri, EMPTY_UUIDS, WALEdit.createFlushWALEdit(hri, f), mvcc, null, sync); if (LOG.isTraceEnabled()) { LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f)); @@ -106,8 +109,8 @@ public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap public static WALKeyImpl writeRegionEventMarker(WAL wal, NavigableMap replicationScope, RegionInfo hri, RegionEventDescriptor r, MultiVersionConcurrencyControl mvcc) throws IOException { - WALKeyImpl walKey = - writeMarker(wal, replicationScope, hri, WALEdit.createRegionEventWALEdit(hri, r), mvcc, null); + WALKeyImpl walKey = writeMarker(wal, replicationScope, hri, EMPTY_UUIDS, + WALEdit.createRegionEventWALEdit(hri, r), mvcc, null); if (LOG.isTraceEnabled()) { LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r)); } @@ -127,10 +130,10 @@ public static WALKeyImpl writeRegionEventMarker(WAL wal, */ public static WALKeyImpl writeBulkLoadMarkerAndSync(final WAL wal, final NavigableMap replicationScope, final RegionInfo hri, - final WALProtos.BulkLoadDescriptor desc, final MultiVersionConcurrencyControl mvcc) - throws IOException { - WALKeyImpl walKey = - writeMarker(wal, replicationScope, hri, WALEdit.createBulkLoadEvent(hri, desc), mvcc, null); + List clusterIds, final WALProtos.BulkLoadDescriptor desc, + final MultiVersionConcurrencyControl mvcc) throws IOException { + WALKeyImpl walKey = writeMarker(wal, replicationScope, hri, clusterIds, + WALEdit.createBulkLoadEvent(hri, desc), mvcc, null); if (LOG.isTraceEnabled()) { LOG.trace("Appended Bulk Load marker " + TextFormat.shortDebugString(desc)); } @@ -138,12 +141,12 @@ public static WALKeyImpl writeBulkLoadMarkerAndSync(final WAL wal, } private static WALKeyImpl writeMarker(final WAL wal, - NavigableMap replicationScope, RegionInfo hri, WALEdit edit, - MultiVersionConcurrencyControl mvcc, Map extendedAttributes) + NavigableMap replicationScope, RegionInfo hri, List clusterIds, + WALEdit edit, MultiVersionConcurrencyControl mvcc, Map extendedAttributes) throws IOException { // If sync == true in below, then timeout is not used; safe to pass UNSPECIFIED_TIMEOUT - return doFullMarkerAppendTransaction(wal, replicationScope, hri, edit, mvcc, extendedAttributes, - true); + return doFullMarkerAppendTransaction(wal, replicationScope, hri, clusterIds, edit, mvcc, + extendedAttributes, true); } /** @@ -155,12 +158,12 @@ private static WALKeyImpl writeMarker(final WAL wal, * @return WALKeyImpl that was added to the WAL. */ private static WALKeyImpl doFullMarkerAppendTransaction(WAL wal, - NavigableMap replicationScope, RegionInfo hri, final WALEdit edit, - MultiVersionConcurrencyControl mvcc, Map extendedAttributes, boolean sync) - throws IOException { + NavigableMap replicationScope, RegionInfo hri, final List clusterIds, + final WALEdit edit, MultiVersionConcurrencyControl mvcc, Map extendedAttributes, + boolean sync) throws IOException { // TODO: Pass in current time to use? WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), - EnvironmentEdgeManager.currentTime(), mvcc, replicationScope, extendedAttributes); + EnvironmentEdgeManager.currentTime(), clusterIds, mvcc, replicationScope, extendedAttributes); long trx = MultiVersionConcurrencyControl.NONE; try { trx = wal.appendMarker(hri, walKey, edit); @@ -232,7 +235,7 @@ public static void writeReplicationMarkerAndSync(WAL wal, MultiVersionConcurrenc RegionInfo regionInfo, byte[] rowKey, long timestamp) throws IOException { NavigableMap replicationScope = new TreeMap<>(Bytes.BYTES_COMPARATOR); replicationScope.put(WALEdit.METAFAMILY, REPLICATION_SCOPE_GLOBAL); - writeMarker(wal, replicationScope, regionInfo, + writeMarker(wal, replicationScope, regionInfo, EMPTY_UUIDS, WALEdit.createReplicationMarkerEdit(rowKey, timestamp), mvcc, null); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java index 961e4d859662..a3f55237c8ff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java @@ -249,8 +249,11 @@ public void replicateEntries(List entries, final CellScanner cells, } // Map of table name Vs list of pair of family and list of // hfile paths from its namespace + + List clusterIds = entry.getKey().getClusterIdsList().stream() + .map(k -> toUUID(k).toString()).collect(Collectors.toList()); Map>>> bulkLoadHFileMap = - bulkLoadsPerClusters.computeIfAbsent(bld.getClusterIdsList(), k -> new HashMap<>()); + bulkLoadsPerClusters.computeIfAbsent(clusterIds, k -> new HashMap<>()); buildBulkLoadHFileMap(bulkLoadHFileMap, table, bld); } } else if (CellUtil.matchingQualifier(cell, WALEdit.REPLICATION_MARKER)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java index aff98431e564..6c6a917b55d0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALKeyImpl.java @@ -152,9 +152,9 @@ public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, fin } public WALKeyImpl(final byte[] encodedRegionName, final TableName tablename, final long now, - MultiVersionConcurrencyControl mvcc, final NavigableMap replicationScope, - Map extendedAttributes) { - init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, EMPTY_UUIDS, HConstants.NO_NONCE, + List clusterIds, MultiVersionConcurrencyControl mvcc, + final NavigableMap replicationScope, Map extendedAttributes) { + init(encodedRegionName, tablename, NO_SEQUENCE_ID, now, clusterIds, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, replicationScope, extendedAttributes); }