Merge branch 'master' into stacicho-pipelines-local-add-notebook

Author: BasilBeirouti

src/sagemaker/clarify.py

@@ -823,6 +823,29 @@ def _prepare_debugger_for_training(self):
             self.debugger_hook_config.s3_output_path = self.output_path
         self.debugger_rule_configs = self._prepare_debugger_rules()
         self._prepare_collection_configs()
+        self._validate_and_set_debugger_configs()
+        if not self.debugger_hook_config:
+            if self.environment is None:
+                self.environment = {}
+            self.environment[DEBUGGER_FLAG] = "0"
+
+    def _validate_and_set_debugger_configs(self):
+        """Set defaults for debugging."""
+        region_supports_debugger = _region_supports_debugger(
+            self.sagemaker_session.boto_region_name
+        )
+
+        if region_supports_debugger:
+            if self.debugger_hook_config in [None, {}]:
+                self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
+        else:
+            if self.debugger_hook_config is not False and self.debugger_hook_config:
+                # when user set debugger config in a unsupported region
+                raise ValueError(
+                    "Current region does not support debugger but debugger hook config is set!"
+                )
+            # disable debugger in unsupported regions
+            self.debugger_hook_config = False
 
     def _prepare_debugger_rules(self):
         """Set any necessary values in debugger rules, if they are provided."""
@@ -1766,6 +1789,8 @@ def enable_default_profiling(self):
         Debugger monitoring is disabled.
         """
         self._ensure_latest_training_job()
+        if not _region_supports_debugger(self.sagemaker_session.boto_region_name):
+            raise ValueError("Current region does not support profiler / debugger!")
 
         training_job_details = self.latest_training_job.describe()
 
@@ -1799,6 +1824,8 @@ def disable_profiling(self):
 
         """
         self._ensure_latest_training_job()
+        if not _region_supports_debugger(self.sagemaker_session.boto_region_name):
+            raise ValueError("Current region does not support profiler / debugger!")
 
         training_job_details = self.latest_training_job.describe()
 
@@ -1852,6 +1879,8 @@ def update_profiler(
 
         """
         self._ensure_latest_training_job()
+        if not _region_supports_debugger(self.sagemaker_session.boto_region_name):
+            raise ValueError("Current region does not support profiler / debugger!")
 
         if (
             not rules
@@ -2872,13 +2901,7 @@ def _script_mode_hyperparam_update(self, code_dir: str, script: str) -> None:
 
     def _validate_and_set_debugger_configs(self):
         """Set defaults for debugging."""
-        if self.debugger_hook_config is None and _region_supports_debugger(
-            self.sagemaker_session.boto_region_name
-        ):
-            self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
-        elif not self.debugger_hook_config:
-            # set hook config to False if _region_supports_debugger is False
-            self.debugger_hook_config = False
+        super(Framework, self)._validate_and_set_debugger_configs()
 
         # Disable debugger if checkpointing is enabled by the customer
         if self.checkpoint_s3_uri and self.checkpoint_local_path and self.debugger_hook_config:
@@ -2901,11 +2924,6 @@ def _validate_and_set_debugger_configs(self):
                     )
                     self.debugger_hook_config = False
 
-        if self.debugger_hook_config is False:
-            if self.environment is None:
-                self.environment = {}
-            self.environment[DEBUGGER_FLAG] = "0"
-
     def _model_source_dir(self):
         """Get the appropriate value to pass as ``source_dir`` to a model constructor.
 

src/sagemaker/estimator.py

@@ -53,8 +53,26 @@
     "only one worker per host regardless of the number of GPUs."
 )
 
-DEBUGGER_UNSUPPORTED_REGIONS = ("us-iso-east-1",)
-PROFILER_UNSUPPORTED_REGIONS = ("us-iso-east-1",)
+DEBUGGER_UNSUPPORTED_REGIONS = (
+    "us-iso-east-1",
+    "ap-southeast-3",
+    "ap-southeast-4",
+    "eu-south-2",
+    "me-central-1",
+    "ap-south-2",
+    "eu-central-2",
+    "us-gov-east-1",
+)
+PROFILER_UNSUPPORTED_REGIONS = (
+    "us-iso-east-1",
+    "ap-southeast-3",
+    "ap-southeast-4",
+    "eu-south-2",
+    "me-central-1",
+    "ap-south-2",
+    "eu-central-2",
+    "us-gov-east-1",
+)
 
 SINGLE_GPU_INSTANCE_TYPES = ("ml.p2.xlarge", "ml.p3.2xlarge")
 SM_DATAPARALLEL_SUPPORTED_INSTANCE_TYPES = (

src/sagemaker/fw_utils.py

@@ -138,7 +138,9 @@ def data_path_no_label_index(training_set_no_label):
 def data_path_label_index(training_set_label_index):
     features, label, index = training_set_label_index
     data = pd.concat(
-        [pd.DataFrame(label), pd.DataFrame(features), pd.DataFrame(index)], axis=1, sort=False
+        [pd.DataFrame(label), pd.DataFrame(features), pd.DataFrame(index)],
+        axis=1,
+        sort=False,
     )
     with tempfile.TemporaryDirectory() as tmpdirname:
         filename = os.path.join(tmpdirname, "train_label_index.csv")
@@ -151,7 +153,12 @@ def data_path_label_index(training_set_label_index):
 def data_path_label_index_6col(training_set_label_index):
     features, label, index = training_set_label_index
     data = pd.concat(
-        [pd.DataFrame(label), pd.DataFrame(features), pd.DataFrame(features), pd.DataFrame(index)],
+        [
+            pd.DataFrame(label),
+            pd.DataFrame(features),
+            pd.DataFrame(features),
+            pd.DataFrame(index),
+        ],
         axis=1,
         sort=False,
     )
@@ -551,7 +558,10 @@ def test_pre_training_bias(clarify_processor, data_config, data_bias_config, sag
 
 
 def test_pre_training_bias_facets_not_included(
-    clarify_processor, data_config_facets_not_included, data_bias_config, sagemaker_session
+    clarify_processor,
+    data_config_facets_not_included,
+    data_bias_config,
+    sagemaker_session,
 ):
     with timeout.timeout(minutes=CLARIFY_DEFAULT_TIMEOUT_MINUTES):
         clarify_processor.run_pre_training_bias(
@@ -643,7 +653,9 @@ def test_post_training_bias_facets_not_included_excluded_columns(
             <= 1.0
         )
         check_analysis_config(
-            data_config_facets_not_included_multiple_files, sagemaker_session, "post_training_bias"
+            data_config_facets_not_included_multiple_files,
+            sagemaker_session,
+            "post_training_bias",
         )
 
 
@@ -704,6 +716,50 @@ def test_shap(clarify_processor, data_config, model_config, shap_config, sagemak
         check_analysis_config(data_config, sagemaker_session, "shap")
 
 
+def test_bias_and_explainability(
+    clarify_processor,
+    data_config,
+    model_config,
+    shap_config,
+    data_bias_config,
+    sagemaker_session,
+):
+    with timeout.timeout(minutes=CLARIFY_DEFAULT_TIMEOUT_MINUTES):
+        clarify_processor.run_bias_and_explainability(
+            data_config,
+            model_config,
+            shap_config,
+            data_bias_config,
+            pre_training_methods="all",
+            post_training_methods="all",
+            model_predicted_label_config="score",
+            job_name=utils.unique_name_from_base("clarify-bias-and-explainability"),
+            wait=True,
+        )
+        analysis_result_json = s3.S3Downloader.read_file(
+            data_config.s3_output_path + "/analysis.json",
+            sagemaker_session,
+        )
+        analysis_result = json.loads(analysis_result_json)
+        assert (
+            math.fabs(
+                analysis_result["explanations"]["kernel_shap"]["label0"]["global_shap_values"]["F2"]
+            )
+            <= 1
+        )
+        check_analysis_config(data_config, sagemaker_session, "shap")
+
+        assert (
+            math.fabs(
+                analysis_result["post_training_bias_metrics"]["facets"]["F1"][0]["metrics"][0][
+                    "value"
+                ]
+            )
+            <= 1.0
+        )
+        check_analysis_config(data_config, sagemaker_session, "post_training_bias")
+
+
 def check_analysis_config(data_config, sagemaker_session, method):
     analysis_config_json = s3.S3Downloader.read_file(
         data_config.s3_output_path + "/analysis_config.json",

tests/integ/test_clarify.py

@@ -483,6 +483,7 @@ def test_fit_ps(time, strftime, sagemaker_session):
     expected_train_args = _create_train_job("1.11", ps=True, py_version="py2")
     expected_train_args["input_config"][0]["DataSource"]["S3DataSource"]["S3Uri"] = inputs
     expected_train_args["hyperparameters"][TensorFlow.LAUNCH_PS_ENV_NAME] = json.dumps(True)
+    expected_train_args["environment"] = {"USE_SMDEBUG": "0"}
 
     actual_train_args = sagemaker_session.method_calls[0][2]
     assert actual_train_args == expected_train_args

tests/unit/sagemaker/tensorflow/test_estimator.py

@@ -370,6 +370,10 @@ def test_training_step_base_estimator(sagemaker_session):
             },
             "RoleArn": ROLE,
             "StoppingCondition": {"MaxRuntimeInSeconds": 86400},
+            "DebugHookConfig": {
+                "S3OutputPath": {"Std:Join": {"On": "/", "Values": ["s3:/", "a", "b"]}},
+                "CollectionConfigurations": [],
+            },
             "ProfilerConfig": {
                 "ProfilingIntervalInMilliseconds": 500,
                 "S3OutputPath": {"Std:Join": {"On": "/", "Values": ["s3:/", "a", "b"]}},