Skip to content

Commit 9d4ac12

Browse files
HADOOP-18325: [ABFS] Fix metric related test failures due to missing config (#6847)
Fixing test failures when the needed configs for metric collection are not set Contributed by: Anmol Asrani
1 parent 07fdc1a commit 9d4ac12

File tree

3 files changed

+46
-13
lines changed

3 files changed

+46
-13
lines changed

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import java.util.concurrent.TimeUnit;
4343
import java.util.concurrent.atomic.AtomicBoolean;
4444

45+
import org.apache.commons.lang3.StringUtils;
4546
import org.apache.hadoop.classification.VisibleForTesting;
4647
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
4748
import org.apache.hadoop.fs.azurebfs.constants.HttpOperationType;
@@ -246,21 +247,24 @@ private AbfsClient(final URL baseUrl,
246247
this.isMetricCollectionStopped = new AtomicBoolean(false);
247248
this.metricAnalysisPeriod = abfsConfiguration.getMetricAnalysisTimeout();
248249
this.metricIdlePeriod = abfsConfiguration.getMetricIdleTimeout();
249-
if (!metricFormat.toString().equals("")) {
250-
isMetricCollectionEnabled = true;
251-
abfsCounters.initializeMetrics(metricFormat);
250+
if (StringUtils.isNotEmpty(metricFormat.toString())) {
252251
String metricAccountName = abfsConfiguration.getMetricAccount();
253-
int dotIndex = metricAccountName.indexOf(AbfsHttpConstants.DOT);
254-
if (dotIndex <= 0) {
255-
throw new InvalidUriException(
256-
metricAccountName + " - account name is not fully qualified.");
257-
}
258252
String metricAccountKey = abfsConfiguration.getMetricAccountKey();
259-
try {
260-
metricSharedkeyCredentials = new SharedKeyCredentials(metricAccountName.substring(0, dotIndex),
261-
metricAccountKey);
262-
} catch (IllegalArgumentException e) {
263-
throw new IOException("Exception while initializing metric credentials " + e);
253+
if (StringUtils.isNotEmpty(metricAccountName) && StringUtils.isNotEmpty(metricAccountKey)) {
254+
isMetricCollectionEnabled = true;
255+
abfsCounters.initializeMetrics(metricFormat);
256+
int dotIndex = metricAccountName.indexOf(AbfsHttpConstants.DOT);
257+
if (dotIndex <= 0) {
258+
throw new InvalidUriException(
259+
metricAccountName + " - account name is not fully qualified.");
260+
}
261+
try {
262+
metricSharedkeyCredentials = new SharedKeyCredentials(
263+
metricAccountName.substring(0, dotIndex),
264+
metricAccountKey);
265+
} catch (IllegalArgumentException e) {
266+
throw new IOException("Exception while initializing metric credentials ", e);
267+
}
264268
}
265269
}
266270
this.timer = new Timer(

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@
2121
import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
2222
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE;
2323
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_WRITE_BUFFER_SIZE;
24+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY;
25+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME;
2426
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_FORMAT;
27+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI;
2528
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
2629
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_KB;
2730
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ONE_MB;
@@ -30,6 +33,8 @@
3033
import org.apache.hadoop.conf.Configuration;
3134
import org.apache.hadoop.fs.FileSystem;
3235
import org.apache.hadoop.fs.azurebfs.utils.MetricFormat;
36+
37+
import org.junit.Assume;
3338
import org.junit.Test;
3439

3540
import java.io.IOException;
@@ -47,6 +52,20 @@
4752
public class ITestAbfsReadFooterMetrics extends AbstractAbfsScaleTest {
4853

4954
public ITestAbfsReadFooterMetrics() throws Exception {
55+
checkPrerequisites();
56+
}
57+
58+
private void checkPrerequisites(){
59+
checkIfConfigIsSet(FS_AZURE_METRIC_ACCOUNT_NAME);
60+
checkIfConfigIsSet(FS_AZURE_METRIC_ACCOUNT_KEY);
61+
checkIfConfigIsSet(FS_AZURE_METRIC_URI);
62+
}
63+
64+
private void checkIfConfigIsSet(String configKey){
65+
AbfsConfiguration conf = getConfiguration();
66+
String value = conf.get(configKey);
67+
Assume.assumeTrue(configKey + " config is mandatory for the test to run",
68+
value != null && value.trim().length() > 1);
5069
}
5170

5271
private static final String TEST_PATH = "/testfile";

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@
2424
import org.apache.hadoop.fs.azurebfs.utils.MetricFormat;
2525
import org.junit.Test;
2626
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_DELETE;
27+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY;
28+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME;
2729
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_FORMAT;
30+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI;
2831
import static org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationType.DeletePath;
2932
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
3033
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
@@ -39,6 +42,12 @@ public class TestAbfsRestOperation extends
3942
public TestAbfsRestOperation() throws Exception {
4043
}
4144

45+
private void checkPrerequisites() {
46+
assumeValidTestConfigPresent(getRawConfiguration(), FS_AZURE_METRIC_ACCOUNT_NAME);
47+
assumeValidTestConfigPresent(getRawConfiguration(), FS_AZURE_METRIC_ACCOUNT_KEY);
48+
assumeValidTestConfigPresent(getRawConfiguration(), FS_AZURE_METRIC_URI);
49+
}
50+
4251
/**
4352
* Test for backoff retry metrics.
4453
*
@@ -49,6 +58,7 @@ public TestAbfsRestOperation() throws Exception {
4958
*/
5059
@Test
5160
public void testBackoffRetryMetrics() throws Exception {
61+
checkPrerequisites();
5262
// Create an AzureBlobFileSystem instance.
5363
final Configuration configuration = getRawConfiguration();
5464
configuration.set(FS_AZURE_METRIC_FORMAT, String.valueOf(MetricFormat.INTERNAL_BACKOFF_METRIC_FORMAT));

0 commit comments

Comments
 (0)