Skip to content

Commit 46995b2

Browse files
guluo2016Apache9
guluo2016
authored and
Apache9
committed
HBASE-28658 The failsafe snapshot should be deleted after rollback successfully (#6019)
Signed-off-by: Duo Zhang <[email protected]> (cherry picked from commit 67d7772)
1 parent 265dcb4 commit 46995b2

File tree

5 files changed

+71
-15
lines changed

5 files changed

+71
-15
lines changed

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2817,6 +2817,13 @@ public void restoreSnapshot(final String snapshotName, final boolean takeFailSaf
28172817
String msg = "Restore snapshot=" + snapshotName + " failed. Rollback to snapshot="
28182818
+ failSafeSnapshotSnapshotName + " succeeded.";
28192819
LOG.error(msg, e);
2820+
try {
2821+
LOG.info("Deleting restore-failsafe snapshot: {}", failSafeSnapshotSnapshotName);
2822+
deleteSnapshot(failSafeSnapshotSnapshotName);
2823+
} catch (IOException deleteSnapshotException) {
2824+
LOG.error("Unable to remove the failsafe snapshot: {}", failSafeSnapshotSnapshotName,
2825+
deleteSnapshotException);
2826+
}
28202827
throw new RestoreSnapshotException(msg, e);
28212828
} catch (IOException ex) {
28222829
String msg = "Failed to restore and rollback to snapshot=" + failSafeSnapshotSnapshotName;

‎hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2148,10 +2148,21 @@ private CompletableFuture<Void> restoreSnapshot(String snapshotName, TableName t
21482148
if (err3 != null) {
21492149
future.completeExceptionally(err3);
21502150
} else {
2151-
String msg =
2152-
"Restore snapshot=" + snapshotName + " failed. Rollback to snapshot="
2153-
+ failSafeSnapshotSnapshotName + " succeeded.";
2154-
future.completeExceptionally(new RestoreSnapshotException(msg, err2));
2151+
// If fail to restore snapshot but rollback successfully, delete the
2152+
// restore-failsafe snapshot.
2153+
LOG.info(
2154+
"Deleting restore-failsafe snapshot: " + failSafeSnapshotSnapshotName);
2155+
addListener(deleteSnapshot(failSafeSnapshotSnapshotName), (ret4, err4) -> {
2156+
if (err4 != null) {
2157+
LOG.error("Unable to remove the failsafe snapshot: {}",
2158+
failSafeSnapshotSnapshotName, err4);
2159+
}
2160+
String msg =
2161+
"Restore snapshot=" + snapshotName + " failed, Rollback to snapshot="
2162+
+ failSafeSnapshotSnapshotName + " succeeded.";
2163+
LOG.error(msg);
2164+
future.completeExceptionally(new RestoreSnapshotException(msg, err2));
2165+
});
21552166
}
21562167
});
21572168
} else {

‎hbase-server/src/test/java/org/apache/hadoop/hbase/client/SnapshotWithAclTestBase.java‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,20 @@
1717
*/
1818
package org.apache.hadoop.hbase.client;
1919

20+
import static org.junit.Assert.assertEquals;
21+
import static org.junit.Assert.assertFalse;
22+
import static org.junit.Assert.assertThrows;
23+
2024
import java.io.IOException;
2125
import java.util.List;
2226
import java.util.regex.Pattern;
2327
import org.apache.hadoop.conf.Configuration;
28+
import org.apache.hadoop.fs.FileSystem;
29+
import org.apache.hadoop.fs.Path;
2430
import org.apache.hadoop.hbase.Coprocessor;
2531
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
2632
import org.apache.hadoop.hbase.HBaseTestingUtility;
33+
import org.apache.hadoop.hbase.HConstants;
2734
import org.apache.hadoop.hbase.TableName;
2835
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
2936
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
@@ -33,6 +40,8 @@
3340
import org.apache.hadoop.hbase.security.access.Permission;
3441
import org.apache.hadoop.hbase.security.access.PermissionStorage;
3542
import org.apache.hadoop.hbase.security.access.SecureTestUtil;
43+
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
44+
import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
3645
import org.apache.hadoop.hbase.util.Bytes;
3746
import org.junit.AfterClass;
3847
import org.junit.Assert;
@@ -110,6 +119,8 @@ public static void setupBeforeClass() throws Exception {
110119
verifyConfiguration(conf);
111120
// Enable EXEC permission checking
112121
conf.setBoolean(AccessControlConstants.EXEC_PERMISSION_CHECKS_KEY, true);
122+
TEST_UTIL.getConfiguration().set(HConstants.SNAPSHOT_RESTORE_FAILSAFE_NAME,
123+
"hbase-failsafe-{snapshot.name}-{restore.timestamp}");
113124
TEST_UTIL.startMiniCluster();
114125
TEST_UTIL.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME);
115126
MasterCoprocessorHost cpHost =
@@ -163,7 +174,7 @@ private void verifyRows(TableName tableName) throws IOException {
163174
byte[] value = result.getValue(TEST_FAMILY, TEST_QUALIFIER);
164175
Assert.assertArrayEquals(value, Bytes.toBytes(rowCount++));
165176
}
166-
Assert.assertEquals(ROW_COUNT, rowCount);
177+
assertEquals(ROW_COUNT, rowCount);
167178
}
168179
}
169180

@@ -172,7 +183,8 @@ private void verifyRows(TableName tableName) throws IOException {
172183
protected abstract void cloneSnapshot(String snapshotName, TableName tableName,
173184
boolean restoreAcl) throws Exception;
174185

175-
protected abstract void restoreSnapshot(String snapshotName, boolean restoreAcl) throws Exception;
186+
protected abstract void restoreSnapshot(String snapshotName, boolean takeFailSafeSnapshot,
187+
boolean restoreAcl) throws Exception;
176188

177189
@Test
178190
public void testRestoreSnapshot() throws Exception {
@@ -215,7 +227,7 @@ public void testRestoreSnapshot() throws Exception {
215227

216228
// restore snapshot with restoreAcl false.
217229
TEST_UTIL.getAdmin().disableTable(TEST_TABLE);
218-
restoreSnapshot(snapshotName1, false);
230+
restoreSnapshot(snapshotName1, false, false);
219231
TEST_UTIL.getAdmin().enableTable(TEST_TABLE);
220232
verifyAllowed(new AccessReadAction(TEST_TABLE), USER_OWNER, USER_RW);
221233
verifyDenied(new AccessReadAction(TEST_TABLE), USER_RO, USER_NONE);
@@ -224,12 +236,36 @@ public void testRestoreSnapshot() throws Exception {
224236

225237
// restore snapshot with restoreAcl true.
226238
TEST_UTIL.getAdmin().disableTable(TEST_TABLE);
227-
restoreSnapshot(snapshotName1, true);
239+
restoreSnapshot(snapshotName1, false, true);
228240
TEST_UTIL.getAdmin().enableTable(TEST_TABLE);
229241
verifyAllowed(new AccessReadAction(TEST_TABLE), USER_OWNER, USER_RO, USER_RW);
230242
verifyDenied(new AccessReadAction(TEST_TABLE), USER_NONE);
231243
verifyAllowed(new AccessWriteAction(TEST_TABLE), USER_OWNER, USER_RW);
232244
verifyDenied(new AccessWriteAction(TEST_TABLE), USER_RO, USER_NONE);
245+
246+
// Delete data.manifest of the snapshot to simulate an invalid snapshot.
247+
Configuration configuration = TEST_UTIL.getConfiguration();
248+
Path rootDir = new Path(configuration.get(HConstants.HBASE_DIR));
249+
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName1, rootDir);
250+
FileSystem fileSystem = FileSystem.get(rootDir.toUri(), configuration);
251+
Path maniFestPath = new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME);
252+
fileSystem.delete(maniFestPath, false);
253+
assertFalse(fileSystem.exists(maniFestPath));
254+
assertEquals(1, TEST_UTIL.getAdmin().listSnapshots(Pattern.compile(snapshotName1)).size());
255+
// There is no failsafe snapshot before restoring.
256+
int failsafeSnapshotCount = TEST_UTIL.getAdmin()
257+
.listSnapshots(Pattern.compile("hbase-failsafe-" + snapshotName1 + ".*")).size();
258+
assertEquals(0, failsafeSnapshotCount);
259+
TEST_UTIL.getAdmin().disableTable(TEST_TABLE);
260+
// We would get Exception when restoring data by this an invalid snapshot.
261+
assertThrows(Exception.class, () -> restoreSnapshot(snapshotName1, true, true));
262+
TEST_UTIL.getAdmin().enableTable(TEST_TABLE);
263+
verifyRows(TEST_TABLE);
264+
// Fail to store snapshot but rollback successfully, so there is no failsafe snapshot after
265+
// restoring.
266+
failsafeSnapshotCount = TEST_UTIL.getAdmin()
267+
.listSnapshots(Pattern.compile("hbase-failsafe-" + snapshotName1 + ".*")).size();
268+
assertEquals(0, failsafeSnapshotCount);
233269
}
234270

235271
final class AccessSnapshotAction implements AccessTestAction {
@@ -257,8 +293,8 @@ public void testDeleteSnapshot() throws Exception {
257293
USER_RO, USER_RW, USER_NONE);
258294
List<SnapshotDescription> snapshotDescriptions =
259295
TEST_UTIL.getAdmin().listSnapshots(Pattern.compile(testSnapshotName));
260-
Assert.assertEquals(1, snapshotDescriptions.size());
261-
Assert.assertEquals(USER_OWNER.getShortName(), snapshotDescriptions.get(0).getOwner());
296+
assertEquals(1, snapshotDescriptions.size());
297+
assertEquals(USER_OWNER.getShortName(), snapshotDescriptions.get(0).getOwner());
262298
AccessTestAction deleteSnapshotAction = () -> {
263299
try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
264300
Admin admin = conn.getAdmin()) {
@@ -271,6 +307,6 @@ public void testDeleteSnapshot() throws Exception {
271307

272308
List<SnapshotDescription> snapshotsAfterDelete =
273309
TEST_UTIL.getAdmin().listSnapshots(Pattern.compile(testSnapshotName));
274-
Assert.assertEquals(0, snapshotsAfterDelete.size());
310+
assertEquals(0, snapshotsAfterDelete.size());
275311
}
276312
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ protected void cloneSnapshot(String snapshotName, TableName tableName, boolean r
4343
}
4444

4545
@Override
46-
protected void restoreSnapshot(String snapshotName, boolean restoreAcl) throws Exception {
47-
TEST_UTIL.getAdmin().restoreSnapshot(snapshotName, false, restoreAcl);
46+
protected void restoreSnapshot(String snapshotName, boolean takeFailSafeSnapshot,
47+
boolean restoreAcl) throws Exception {
48+
TEST_UTIL.getAdmin().restoreSnapshot(snapshotName, takeFailSafeSnapshot, restoreAcl);
4849
}
4950
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAclAsyncAdmin.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,11 @@ protected void cloneSnapshot(String snapshotName, TableName tableName, boolean r
4949
}
5050

5151
@Override
52-
protected void restoreSnapshot(String snapshotName, boolean restoreAcl) throws Exception {
52+
protected void restoreSnapshot(String snapshotName, boolean takeFailSafeSnapshot,
53+
boolean restoreAcl) throws Exception {
5354
try (AsyncConnection conn =
5455
ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get()) {
55-
conn.getAdmin().restoreSnapshot(snapshotName, false, restoreAcl).get();
56+
conn.getAdmin().restoreSnapshot(snapshotName, takeFailSafeSnapshot, restoreAcl).get();
5657
}
5758
}
5859
}

0 commit comments

Comments
 (0)