From c9d6eac0d1ab0cafb851d1548bb1ef722b40e0bf Mon Sep 17 00:00:00 2001 From: chenxu14 Date: Tue, 30 Apr 2019 12:18:26 +0800 Subject: [PATCH 1/3] HBASE-22335 do add hfile ref only when replication_scope is 1 --- .../hbase/replication/regionserver/ReplicationObserver.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java index b7e437f46241..f99ed754f693 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java @@ -60,6 +60,8 @@ public void preCommitStoreFile(final ObserverContext Date: Fri, 10 May 2019 16:43:21 +0800 Subject: [PATCH 2/3] HBASE-22335 addendum --- .../replication/regionserver/Replication.java | 2 +- .../regionserver/ReplicationObserver.java | 2 +- .../hbase/regionserver/TestHRegion.java | 73 +++++++++++++++---- 3 files changed, 61 insertions(+), 16 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java index 9a449ad21c2e..8f513d096dc3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java @@ -226,7 +226,7 @@ public ReplicationSourceManager getReplicationManager() { return this.replicationManager; } - void addHFileRefsToQueue(TableName tableName, byte[] family, List> pairs) + public void addHFileRefsToQueue(TableName tableName, byte[] family, List> pairs) throws IOException { try { this.replicationManager.addHFileRefs(tableName, family, pairs); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java index f99ed754f693..a6790f6f5b24 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java @@ -72,7 +72,7 @@ public void preCommitStoreFile(final ObserverContext> familyPaths = new ArrayList<>(); + HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache( + TEST_UTIL.getConfiguration()); + File hFileLocation = testFolder.newFile(); + try (FSDataOutputStream out = new FSDataOutputStream( + new FileOutputStream(hFileLocation), null)) { + hFileFactory.withOutputStream(out); + hFileFactory.withFileContext(new HFileContext()); + HFile.Writer writer = hFileFactory.create(); + try { + byte[] randomBytes = new byte[100]; + writer.append(new KeyValue(CellUtil.createCell(randomBytes, fam1, randomBytes, + 0L, KeyValue.Type.Put.getCode(), randomBytes))); + } finally { + writer.close(); + } + } + familyPaths.add(new Pair<>(fam1, hFileLocation.getAbsoluteFile().getAbsolutePath())); + region.bulkLoadHFiles(familyPaths, false, null); + } + /** * The same as HRegion class, the only difference is that instantiateHStore will * create a different HStore - HStoreForTesting. [HBASE-8518] From c01432172cb96486df979498ef988a186a95d4b6 Mon Sep 17 00:00:00 2001 From: chenxu14 Date: Mon, 13 May 2019 14:57:28 +0800 Subject: [PATCH 3/3] HBASE-22335 fix checkstyle and whitespace --- .../regionserver/ReplicationObserver.java | 1 - .../hbase/regionserver/TestHRegion.java | 30 +++++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java index a6790f6f5b24..1bba42633fff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationObserver.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionObserver; -import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 4d5fed1ccd53..81e16eb4e2bb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -146,13 +146,7 @@ import org.apache.hadoop.hbase.replication.regionserver.Replication; import org.apache.hadoop.hbase.replication.regionserver.ReplicationObserver; import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; + import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; @@ -174,11 +168,7 @@ import org.apache.hadoop.hbase.wal.WALProvider; import org.apache.hadoop.hbase.wal.WALProvider.Writer; import org.apache.hadoop.hbase.wal.WALSplitter; -import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; -import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; -import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; -import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; + import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -197,6 +187,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.apache.hbase.thirdparty.com.google.protobuf.ByteString; +import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; +import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; +import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.FlushAction; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor.StoreFlushDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.StoreDescriptor; + /** * Basic stand-alone testing of HRegion. No clusters! * @@ -6288,7 +6292,7 @@ public void testBulkLoadReplicationEnabled() throws IOException { getCoprocessors().contains(ReplicationObserver.class.getSimpleName())); } - @Test + @Test public void testSkipAddHFileRefNode() throws IOException { // test for HBASE-22335 TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42);