HADOOP-17076: ABFS: Delegation SAS Generator Updates

Contributed by Thomas Marquardt.

DETAILS:
1) The authentication version in the service has been updated from Dec19 to Feb20, so need to update the client.
2) Add support and test cases for getXattr and setXAttr.
3) Update DelegationSASGenerator and related to use Duration instead of int for time periods.
4) Cleanup DelegationSASGenerator switch/case statement that maps operations to permissions.
5) Cleanup SASGenerator classes to use String.equals instead of ==.

TESTS:
Added tests for getXAttr and setXAttr.

All tests are passing against my account in eastus2euap:

 $mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=8 clean verify
 Tests run: 76, Failures: 0, Errors: 0, Skipped: 0
 Tests run: 441, Failures: 0, Errors: 0, Skipped: 33
 Tests run: 206, Failures: 0, Errors: 0, Skipped: 24

Author: ThomasMarquardt

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java

@@ -33,14 +33,14 @@
 public interface SASTokenProvider {
 
   String CHECK_ACCESS_OPERATION = "check-access";
+  String CREATE_DIRECTORY_OPERATION = "create-directory";
   String CREATE_FILE_OPERATION = "create-file";
   String DELETE_OPERATION = "delete";
   String DELETE_RECURSIVE_OPERATION = "delete-recursive";
   String GET_ACL_OPERATION = "get-acl";
   String GET_STATUS_OPERATION = "get-status";
   String GET_PROPERTIES_OPERATION = "get-properties";
   String LIST_OPERATION = "list";
-  String CREATE_DIRECTORY_OPERATION = "create-directory";
   String READ_OPERATION = "read";
   String RENAME_SOURCE_OPERATION = "rename-source";
   String RENAME_DESTINATION_OPERATION = "rename-destination";

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java

@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.UUID;
 
+import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -365,4 +366,19 @@ public void testRootPath() throws Exception {
     }
     assertEquals(0, count);
   }
+
+  @Test
+  // Test filesystem operations getXAttr and setXAttr
+  public void testProperties() throws Exception {
+    final AzureBlobFileSystem fs = getFileSystem();
+    Path reqPath = new Path(UUID.randomUUID().toString());
+
+    fs.create(reqPath).close();
+
+    final String propertyName = "user.mime_type";
+    final byte[] propertyValue = "text/plain".getBytes("utf-8");
+    fs.setXAttr(reqPath, propertyName, propertyValue);
+
+    assertArrayEquals(propertyValue, fs.getXAttr(reqPath, propertyName));
+  }
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java

@@ -55,8 +55,8 @@ public void initialize(Configuration configuration, String accountName) throws I
     String appSecret = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET);
     String sktid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID);
     String skoid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID);
-    String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minusSeconds(SASGenerator.FIVE_MINUTES));
-    String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plusSeconds(SASGenerator.ONE_DAY));
+    String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minus(SASGenerator.FIVE_MINUTES));
+    String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plus(SASGenerator.ONE_DAY));
     String skv = SASGenerator.AuthenticationVersion.Dec19.toString();
 
     byte[] key = getUserDelegationKey(accountName, appID, appSecret, sktid, skt, ske, skv);

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java

@@ -29,12 +29,12 @@
  * Test Delegation SAS generator.
  */
 public class DelegationSASGenerator extends SASGenerator {
-  private String skoid;
-  private String sktid;
-  private String skt;
-  private String ske;
+  private final String skoid;
+  private final String sktid;
+  private final String skt;
+  private final String ske;
   private final String sks = "b";
-  private String skv;
+  private final String skv;
 
   public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String sktid, String skt, String ske, String skv) {
     super(userDelegationKey);
@@ -48,20 +48,18 @@ public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String skt
   public String getDelegationSAS(String accountName, String containerName, String path, String operation,
                                  String saoid, String suoid, String scid) {
 
-    final String sv = AuthenticationVersion.Dec19.toString();
-    final String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES));
-    final String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY));
+    final String sv = AuthenticationVersion.Feb20.toString();
+    final String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES));
+    final String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY));
     String sr = "b";
     String sdd = null;
-    String sp = null;
+    String sp;
 
     switch (operation) {
-      case SASTokenProvider.CHECK_ACCESS_OPERATION:
-        sp = "e";
-        break;
-      case SASTokenProvider.WRITE_OPERATION:
       case SASTokenProvider.CREATE_FILE_OPERATION:
       case SASTokenProvider.CREATE_DIRECTORY_OPERATION:
+      case SASTokenProvider.WRITE_OPERATION:
+      case SASTokenProvider.SET_PROPERTIES_OPERATION:
         sp = "w";
         break;
       case SASTokenProvider.DELETE_OPERATION:
@@ -72,13 +70,15 @@ public String getDelegationSAS(String accountName, String containerName, String
         sr = "d";
         sdd = Integer.toString(StringUtils.countMatches(path, "/"));
         break;
+      case SASTokenProvider.CHECK_ACCESS_OPERATION:
       case SASTokenProvider.GET_ACL_OPERATION:
       case SASTokenProvider.GET_STATUS_OPERATION:
         sp = "e";
         break;
       case SASTokenProvider.LIST_OPERATION:
         sp = "l";
         break;
+      case SASTokenProvider.GET_PROPERTIES_OPERATION:
       case SASTokenProvider.READ_OPERATION:
         sp = "r";
         break;
@@ -87,14 +87,12 @@ public String getDelegationSAS(String accountName, String containerName, String
         sp = "m";
         break;
       case SASTokenProvider.SET_ACL_OPERATION:
+      case SASTokenProvider.SET_PERMISSION_OPERATION:
         sp = "p";
         break;
       case SASTokenProvider.SET_OWNER_OPERATION:
         sp = "o";
         break;
-      case SASTokenProvider.SET_PERMISSION_OPERATION:
-        sp = "p";
-        break;
       default:
         throw new IllegalArgumentException(operation);
     }
@@ -146,7 +144,7 @@ private String computeSignatureForSAS(String sp, String st, String se, String sv
     sb.append(accountName);
     sb.append("/");
     sb.append(containerName);
-    if (path != null && sr != "c") {
+    if (path != null && !sr.equals("c")) {
       sb.append(path);
     }
     sb.append("\n");

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java

@@ -20,6 +20,7 @@
 
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.time.format.DateTimeFormatter;
 import java.time.ZoneId;
 import java.util.Locale;
@@ -35,7 +36,8 @@ public abstract class SASGenerator {
 
   public enum AuthenticationVersion {
     Nov18("2018-11-09"),
-    Dec19("2019-12-12");
+    Dec19("2019-12-12"),
+    Feb20("2020-02-10");
 
     private final String ver;
 
@@ -50,8 +52,8 @@ public String toString() {
   }
 
   protected static final Logger LOG = LoggerFactory.getLogger(SASGenerator.class);
-  public static final int FIVE_MINUTES = 5 * 60;
-  public static final int ONE_DAY = 24 * 60 * 60;
+  public static final Duration FIVE_MINUTES = Duration.ofMinutes(5);
+  public static final Duration ONE_DAY = Duration.ofDays(1);
   public static final DateTimeFormatter ISO_8601_FORMATTER =
       DateTimeFormatter
           .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT)

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java

@@ -40,8 +40,8 @@ public String getContainerSASWithFullControl(String accountName, String containe
     String sp = "rcwdl";
     String sv = AuthenticationVersion.Nov18.toString();
     String sr = "c";
-    String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES));
-    String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY));
+    String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES));
+    String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY));
 
     String signature = computeSignatureForSAS(sp, st, se, sv, "c",
         accountName, containerName, null);
@@ -71,7 +71,7 @@ private String computeSignatureForSAS(String sp, String st, String se, String sv
     sb.append(accountName);
     sb.append("/");
     sb.append(containerName);
-    if (path != null && sr != "c") {
+    if (path != null && !sr.equals("c")) {
       //sb.append("/");
       sb.append(path);
     }