added run_bias_and_explainability method

Author: aws-byeldos

src/sagemaker/clarify.py

@@ -1369,15 +1369,139 @@ def run_explainability(
             experiment_config,
         )
 
-    def run_bias_and_explainability(self):
-        """
-        TODO:
-        - add doc string
-        - add logic
-        - add tests
-        """
-        raise NotImplementedError(
-            "Please choose a method of run_pre_training_bias, run_post_training_bias or run_explainability."
+    def run_bias_and_explainability(
+        self,
+        data_config: DataConfig,
+        model_config: ModelConfig,
+        explainability_config: Union[ExplainabilityConfig, List[ExplainabilityConfig]],
+        bias_config: BiasConfig,
+        pre_training_methods: Union[str, List[str]] = "all",
+        post_training_methods: Union[str, List[str]] = "all",
+        model_predicted_label_config: ModelPredictedLabelConfig = None,
+        wait=True,
+        logs=True,
+        job_name=None,
+        kms_key=None,
+        experiment_config=None,
+    ):
+        """Runs a :class:`~sagemaker.processing.ProcessingJob` computing feature attributions.
+
+        For bias:
+        Computes metrics for both the pre-training and the post-training methods.
+        To calculate post-training methods, it spins up a model endpoint and runs inference over the
+        input examples in 's3_data_input_path' (from the :class:`~sagemaker.clarify.DataConfig`)
+        to obtain predicted labels.
+
+        For Explainability:
+        Spins up a model endpoint.
+
+        Currently, only SHAP and  Partial Dependence Plots (PDP) are supported
+        as explainability methods.
+        You can request both methods or one at a time with the ``explainability_config`` parameter.
+
+        When SHAP is requested in the ``explainability_config``,
+        the SHAP algorithm calculates the feature importance for each input example
+        in the ``s3_data_input_path`` of the :class:`~sagemaker.clarify.DataConfig`,
+        by creating ``num_samples`` copies of the example with a subset of features
+        replaced with values from the ``baseline``.
+        It then runs model inference to see how the model's prediction changes with the replaced
+        features. If the model output returns multiple scores importance is computed for each score.
+        Across examples, feature importance is aggregated using ``agg_method``.
+
+        When PDP is requested in the ``explainability_config``,
+        the PDP algorithm calculates the dependence of the target response
+        on the input features and marginalizes over the values of all other input features.
+        The Partial Dependence Plots are included in the output
+        `report <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-feature-attribute-baselines-reports.html>`__
+        and the corresponding values are included in the analysis output.
+
+        Args:
+            data_config (:class:`~sagemaker.clarify.DataConfig`): Config of the input/output data.
+            model_config (:class:`~sagemaker.clarify.ModelConfig`): Config of the model and its
+                endpoint to be created.
+            explainability_config (:class:`~sagemaker.clarify.ExplainabilityConfig` or list):
+                Config of the specific explainability method or a list of
+                :class:`~sagemaker.clarify.ExplainabilityConfig` objects.
+                Currently, SHAP and PDP are the two methods supported.
+                You can request multiple methods at once by passing in a list of
+                `~sagemaker.clarify.ExplainabilityConfig`.
+            bias_config (:class:`~sagemaker.clarify.BiasConfig`): Config of sensitive groups.
+            pre_training_methods (str or list[str]): Selector of a subset of potential metrics:
+                ["`CI <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-bias-metric-class-imbalance.html>`_",
+                "`DPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-true-label-imbalance.html>`_",
+                "`KL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-kl-divergence.html>`_",
+                "`JS <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-jensen-shannon-divergence.html>`_",
+                "`LP <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-lp-norm.html>`_",
+                "`TVD <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-total-variation-distance.html>`_",
+                "`KS <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-kolmogorov-smirnov.html>`_",
+                "`CDDL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-cddl.html>`_"].
+                Defaults to str "all" to run all metrics if left unspecified.
+            post_training_methods (str or list[str]): Selector of a subset of potential metrics:
+                ["`DPPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dppl.html>`_"
+                , "`DI <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-di.html>`_",
+                "`DCA <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dca.html>`_",
+                "`DCR <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dcr.html>`_",
+                "`RD <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-rd.html>`_",
+                "`DAR <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dar.html>`_",
+                "`DRR <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-drr.html>`_",
+                "`AD <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-ad.html>`_",
+                "`CDDPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-cddpl.html>`_
+                ", "`TE <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-te.html>`_",
+                "`FT <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-ft.html>`_"].
+                Defaults to str "all" to run all metrics if left unspecified.
+            model_predicted_label_config (
+                int or 
+                str or 
+                :class:`~sagemaker.clarify.ModelPredictedLabelConfig`
+            ):
+                Index or JSONPath to locate the predicted scores in the model output. This is not
+                required if the model output is a single score. Alternatively, it can be an instance
+                of :class:`~sagemaker.clarify.SageMakerClarifyProcessor`
+                to provide more parameters like ``label_headers``.
+            wait (bool): Whether the call should wait until the job completes (default: True).
+            logs (bool): Whether to show the logs produced by the job.
+                Only meaningful when ``wait`` is True (default: True).
+            job_name (str): Processing job name. When ``job_name`` is not specified,
+                if ``job_name_prefix`` in :class:`~sagemaker.clarify.SageMakerClarifyProcessor`
+                is specified, the job name will be composed of ``job_name_prefix`` and current
+                timestamp; otherwise use ``"Clarify-Explainability"`` as prefix.
+            kms_key (str): The ARN of the KMS key that is used to encrypt the
+                user code file (default: None).
+            experiment_config (dict[str, str]): Experiment management configuration.
+                Optionally, the dict can contain three keys:
+                ``'ExperimentName'``, ``'TrialName'``, and ``'TrialComponentDisplayName'``.
+
+                The behavior of setting these keys is as follows:
+
+                * If ``'ExperimentName'`` is supplied but ``'TrialName'`` is not, a Trial will be
+                  automatically created and the job's Trial Component associated with the Trial.
+                * If ``'TrialName'`` is supplied and the Trial already exists,
+                  the job's Trial Component will be associated with the Trial.
+                * If both ``'ExperimentName'`` and ``'TrialName'`` are not supplied,
+                  the Trial Component will be unassociated.
+                * ``'TrialComponentDisplayName'`` is used for display in Amazon SageMaker Studio.
+        """  # noqa E501  # pylint: disable=c0301
+        analysis_config = _AnalysisConfigGenerator.bias_and_explainability(
+            data_config,
+            model_config,
+            model_predicted_label_config,
+            explainability_config,
+            bias_config,
+            pre_training_methods,
+            post_training_methods,
+        )
+        # when name is either not provided (is None) or an empty string ("")
+        job_name = job_name or utils.name_from_base(
+            self.job_name_prefix or "Clarify-Bias-And-Explainability"
+        )
+        return self._run(
+            data_config,
+            analysis_config,
+            wait,
+            logs,
+            job_name,
+            kms_key,
+            experiment_config,
         )
 
 
@@ -1395,6 +1519,7 @@ def bias_and_explainability(
         pre_training_methods: Union[str, List[str]] = "all",
         post_training_methods: Union[str, List[str]] = "all",
     ):
+        """Generates a config for Bias and Explainability"""
         analysis_config = {**data_config.get_config(), **bias_config.get_config()}
         analysis_config = cls._add_methods(
             analysis_config,
@@ -1475,6 +1600,7 @@ def bias(
 
     @classmethod
     def _add_predictor(cls, analysis_config, model_config, model_predicted_label_config):
+        """Extends analysis config with predictor."""
         analysis_config = {**analysis_config}
         analysis_config["predictor"] = model_config.get_predictor_config()
         if isinstance(model_predicted_label_config, ModelPredictedLabelConfig):
@@ -1498,12 +1624,14 @@ def _add_methods(
         explainability_config=None,
         report=True,
     ):
+        """Extends analysis config with methods."""
         # validate
         params = [pre_training_methods, post_training_methods, explainability_config]
         if all([1 if p is None else 0 for p in params]):
             raise AttributeError(
                 "analysis_config must have at least one working method: "
-                "One of the `pre_training_methods`, `post_training_methods`, `explainability_config`."
+                "One of the "
+                "`pre_training_methods`, `post_training_methods`, `explainability_config`."
             )
 
         # main logic
@@ -1529,6 +1657,7 @@ def _add_methods(
     def _merge_explainability_configs(
         cls, explainability_config: Union[ExplainabilityConfig, List[ExplainabilityConfig]]
     ):
+        """Merges explainability configs, when more than one."""
         if isinstance(explainability_config, list):
             explainability_methods = {}
             if len(explainability_config) == 0:

tests/integ/test_clarify.py

@@ -704,6 +704,45 @@ def test_shap(clarify_processor, data_config, model_config, shap_config, sagemak
         check_analysis_config(data_config, sagemaker_session, "shap")
 
 
+def test_bias_and_explainability(
+    clarify_processor, data_config, model_config, shap_config, data_bias_config, sagemaker_session
+):
+    with timeout.timeout(minutes=CLARIFY_DEFAULT_TIMEOUT_MINUTES):
+        clarify_processor.run_bias_and_explainability(
+            data_config,
+            model_config,
+            shap_config,
+            data_bias_config,
+            pre_training_methods="all",
+            post_training_methods="all",
+            model_predicted_label_config="score",
+            job_name=utils.unique_name_from_base("clarify-bias-and-explainability"),
+            wait=True,
+        )
+        analysis_result_json = s3.S3Downloader.read_file(
+            data_config.s3_output_path + "/analysis.json",
+            sagemaker_session,
+        )
+        analysis_result = json.loads(analysis_result_json)
+        assert (
+            math.fabs(
+                analysis_result["explanations"]["kernel_shap"]["label0"]["global_shap_values"]["F2"]
+            )
+            <= 1
+        )
+        check_analysis_config(data_config, sagemaker_session, "shap")
+
+        assert (
+            math.fabs(
+                analysis_result["post_training_bias_metrics"]["facets"]["F1"][0]["metrics"][0][
+                    "value"
+                ]
+            )
+            <= 1.0
+        )
+        check_analysis_config(data_config, sagemaker_session, "post_training_bias")
+
+
 def check_analysis_config(data_config, sagemaker_session, method):
     analysis_config_json = s3.S3Downloader.read_file(
         data_config.s3_output_path + "/analysis_config.json",