HADOOP-17714 ABFS: testBlobBackCompatibility, testRandomRead & WasbAbfsCompatibility tests fail when triggered with default configs (apache#3035)

Author: snehavarma

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java

@@ -149,7 +149,7 @@ public void setup() throws Exception {
     if (!isIPAddress
         && (abfsConfig.getAuthType(accountName) != AuthType.SAS)
         && !abfs.getIsNamespaceEnabled()) {
-      final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl()));
+      final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl(), abfsConfig.isHttpsAlwaysUsed()));
       final AzureNativeFileSystemStore azureNativeFileSystemStore =
           new AzureNativeFileSystemStore();
 
@@ -355,13 +355,13 @@ protected void touch(Path path) throws IOException {
   protected static String wasbUrlToAbfsUrl(final String wasbUrl) {
     return convertTestUrls(
         wasbUrl, FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX,
-        FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX);
+        FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX, false);
   }
 
-  protected static String abfsUrlToWasbUrl(final String abfsUrl) {
+  protected static String abfsUrlToWasbUrl(final String abfsUrl, final boolean isAlwaysHttpsUsed) {
     return convertTestUrls(
         abfsUrl, FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX,
-        FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX);
+        FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX, isAlwaysHttpsUsed);
   }
 
   private static String convertTestUrls(
@@ -371,15 +371,17 @@ private static String convertTestUrls(
       final String fromDnsPrefix,
       final String toNonSecureScheme,
       final String toSecureScheme,
-      final String toDnsPrefix) {
+      final String toDnsPrefix,
+      final boolean isAlwaysHttpsUsed) {
     String data = null;
-    if (url.startsWith(fromNonSecureScheme + "://")) {
+    if (url.startsWith(fromNonSecureScheme + "://") && isAlwaysHttpsUsed) {
+      data = url.replace(fromNonSecureScheme + "://", toSecureScheme + "://");
+    } else if (url.startsWith(fromNonSecureScheme + "://")) {
       data = url.replace(fromNonSecureScheme + "://", toNonSecureScheme + "://");
     } else if (url.startsWith(fromSecureScheme + "://")) {
       data = url.replace(fromSecureScheme + "://", toSecureScheme + "://");
     }
 
-
     if (data != null) {
       data = data.replace("." + fromDnsPrefix + ".",
           "." + toDnsPrefix + ".");

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java

@@ -74,6 +74,12 @@ private String getBlobConnectionString() {
               + ";AccountName=" + this.getAccountName().split("\\.")[0]
               + ";AccountKey=" + this.getAccountKey();
     }
+    else if (this.getConfiguration().isHttpsAlwaysUsed()) {
+      connectionString = "DefaultEndpointsProtocol=https;BlobEndpoint=https://"
+              + this.getAccountName().replaceFirst("\\.dfs\\.", ".blob.")
+              + ";AccountName=" + this.getAccountName().split("\\.")[0]
+              + ";AccountKey=" + this.getAccountKey();
+    }
     else {
       connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://"
               + this.getAccountName().replaceFirst("\\.dfs\\.", ".blob.")

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java

@@ -159,7 +159,7 @@ public void testUrlConversion(){
     String abfsUrl = "abfs://[email protected]";
     String wabsUrl = "wasb://[email protected]";
     assertEquals(abfsUrl, wasbUrlToAbfsUrl(wabsUrl));
-    assertEquals(wabsUrl, abfsUrlToWasbUrl(abfsUrl));
+    assertEquals(wabsUrl, abfsUrlToWasbUrl(abfsUrl, false));
   }
 
   @Test