added run_bias_and_explainability method

aws-byeldos · aws-byeldos · commit 8eb4b013d9d1 · 2022-08-08T17:48:58.000+02:00
diff --git a/src/sagemaker/clarify.py b/src/sagemaker/clarify.py
@@ -1368,15 +1368,135 @@ def run_explainability(
             experiment_config,
         )
 
-    def run_bias_and_explainability(self):
-        """
-        TODO:
-        - add doc string
-        - add logic
-        - add tests
-        """
-        raise NotImplementedError(
-            "Please choose a method of run_pre_training_bias, run_post_training_bias or run_explainability."
+    def run_bias_and_explainability(
+        self,
+        data_config: DataConfig,
+        model_config: ModelConfig,
+        explainability_config: Union[ExplainabilityConfig, List[ExplainabilityConfig]],
+        bias_config: BiasConfig,
+        pre_training_methods: Union[str, List[str]] = "all",
+        post_training_methods: Union[str, List[str]] = "all",
+        model_predicted_label_config: ModelPredictedLabelConfig = None,
+        wait=True,
+        logs=True,
+        job_name=None,
+        kms_key=None,
+        experiment_config=None,
+    ):
+        """Runs a :class:`~sagemaker.processing.ProcessingJob` computing feature attributions.
+
+        For bias:
+        Computes metrics for both the pre-training and the post-training methods.
+        To calculate post-training methods, it spins up a model endpoint and runs inference over the
+        input examples in 's3_data_input_path' (from the :class:`~sagemaker.clarify.DataConfig`)
+        to obtain predicted labels.
+
+        For Explainability:
+        Spins up a model endpoint.
+
+        Currently, only SHAP and  Partial Dependence Plots (PDP) are supported
+        as explainability methods.
+        You can request both methods or one at a time with the ``explainability_config`` parameter.
+
+        When SHAP is requested in the ``explainability_config``,
+        the SHAP algorithm calculates the feature importance for each input example
+        in the ``s3_data_input_path`` of the :class:`~sagemaker.clarify.DataConfig`,
+        by creating ``num_samples`` copies of the example with a subset of features
+        replaced with values from the ``baseline``.
+        It then runs model inference to see how the model's prediction changes with the replaced
+        features. If the model output returns multiple scores importance is computed for each score.
+        Across examples, feature importance is aggregated using ``agg_method``.
+
+        When PDP is requested in the ``explainability_config``,
+        the PDP algorithm calculates the dependence of the target response
+        on the input features and marginalizes over the values of all other input features.
+        The Partial Dependence Plots are included in the output
+        `report <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-feature-attribute-baselines-reports.html>`__
+        and the corresponding values are included in the analysis output.
+
+        Args:
+            data_config (:class:`~sagemaker.clarify.DataConfig`): Config of the input/output data.
+            model_config (:class:`~sagemaker.clarify.ModelConfig`): Config of the model and its
+                endpoint to be created.
+            explainability_config (:class:`~sagemaker.clarify.ExplainabilityConfig` or list):
+                Config of the specific explainability method or a list of
+                :class:`~sagemaker.clarify.ExplainabilityConfig` objects.
+                Currently, SHAP and PDP are the two methods supported.
+                You can request multiple methods at once by passing in a list of
+                `~sagemaker.clarify.ExplainabilityConfig`.
+            bias_config (:class:`~sagemaker.clarify.BiasConfig`): Config of sensitive groups.
+            pre_training_methods (str or list[str]): Selector of a subset of potential metrics:
+                ["`CI <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-bias-metric-class-imbalance.html>`_",
+                "`DPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-true-label-imbalance.html>`_",
+                "`KL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-kl-divergence.html>`_",
+                "`JS <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-jensen-shannon-divergence.html>`_",
+                "`LP <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-lp-norm.html>`_",
+                "`TVD <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-total-variation-distance.html>`_",
+                "`KS <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-kolmogorov-smirnov.html>`_",
+                "`CDDL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-data-bias-metric-cddl.html>`_"].
+                Defaults to str "all" to run all metrics if left unspecified.
+            post_training_methods (str or list[str]): Selector of a subset of potential metrics:
+                ["`DPPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dppl.html>`_"
+                , "`DI <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-di.html>`_",
+                "`DCA <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dca.html>`_",
+                "`DCR <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dcr.html>`_",
+                "`RD <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-rd.html>`_",
+                "`DAR <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-dar.html>`_",
+                "`DRR <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-drr.html>`_",
+                "`AD <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-ad.html>`_",
+                "`CDDPL <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-cddpl.html>`_
+                ", "`TE <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-te.html>`_",
+                "`FT <https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-post-training-bias-metric-ft.html>`_"].
+                Defaults to str "all" to run all metrics if left unspecified.
+            model_predicted_label_config (int or str or :class:`~sagemaker.clarify.ModelPredictedLabelConfig`):
+                Index or JSONPath to locate the predicted scores in the model output. This is not
+                required if the model output is a single score. Alternatively, it can be an instance
+                of :class:`~sagemaker.clarify.SageMakerClarifyProcessor`
+                to provide more parameters like ``label_headers``.
+            wait (bool): Whether the call should wait until the job completes (default: True).
+            logs (bool): Whether to show the logs produced by the job.
+                Only meaningful when ``wait`` is True (default: True).
+            job_name (str): Processing job name. When ``job_name`` is not specified,
+                if ``job_name_prefix`` in :class:`~sagemaker.clarify.SageMakerClarifyProcessor`
+                is specified, the job name will be composed of ``job_name_prefix`` and current
+                timestamp; otherwise use ``"Clarify-Explainability"`` as prefix.
+            kms_key (str): The ARN of the KMS key that is used to encrypt the
+                user code file (default: None).
+            experiment_config (dict[str, str]): Experiment management configuration.
+                Optionally, the dict can contain three keys:
+                ``'ExperimentName'``, ``'TrialName'``, and ``'TrialComponentDisplayName'``.
+
+                The behavior of setting these keys is as follows:
+
+                * If ``'ExperimentName'`` is supplied but ``'TrialName'`` is not, a Trial will be
+                  automatically created and the job's Trial Component associated with the Trial.
+                * If ``'TrialName'`` is supplied and the Trial already exists,
+                  the job's Trial Component will be associated with the Trial.
+                * If both ``'ExperimentName'`` and ``'TrialName'`` are not supplied,
+                  the Trial Component will be unassociated.
+                * ``'TrialComponentDisplayName'`` is used for display in Amazon SageMaker Studio.
+        """  # noqa E501  # pylint: disable=c0301
+        analysis_config = _AnalysisConfigGenerator.bias_and_explainability(
+            data_config,
+            model_config,
+            model_predicted_label_config,
+            explainability_config,
+            bias_config,
+            pre_training_methods,
+            post_training_methods,
+        )
+        # when name is either not provided (is None) or an empty string ("")
+        job_name = job_name or utils.name_from_base(
+            self.job_name_prefix or "Clarify-Bias-And-Explainability"
+        )
+        return self._run(
+            data_config,
+            analysis_config,
+            wait,
+            logs,
+            job_name,
+            kms_key,
+            experiment_config,
         )
 
 
diff --git a/tests/integ/test_clarify.py b/tests/integ/test_clarify.py
@@ -704,6 +704,50 @@ def test_shap(clarify_processor, data_config, model_config, shap_config, sagemak
         check_analysis_config(data_config, sagemaker_session, "shap")
 
 
+def test_bias_and_explainability(
+        clarify_processor,
+        data_config,
+        model_config,
+        shap_config,
+        data_bias_config,
+        sagemaker_session
+    ):
+    with timeout.timeout(minutes=CLARIFY_DEFAULT_TIMEOUT_MINUTES):
+        clarify_processor.run_bias_and_explainability(
+            data_config,
+            model_config,
+            shap_config,
+            data_bias_config,
+            pre_training_methods="all",
+            post_training_methods="all",
+            model_predicted_label_config="score",
+            job_name=utils.unique_name_from_base("clarify-bias-and-explainability"),
+            wait=True,
+        )
+        analysis_result_json = s3.S3Downloader.read_file(
+            data_config.s3_output_path + "/analysis.json",
+            sagemaker_session,
+        )
+        analysis_result = json.loads(analysis_result_json)
+        assert (
+            math.fabs(
+                analysis_result["explanations"]["kernel_shap"]["label0"]["global_shap_values"]["F2"]
+            )
+            <= 1
+        )
+        check_analysis_config(data_config, sagemaker_session, "shap")
+
+        assert (
+                math.fabs(
+                    analysis_result["post_training_bias_metrics"]["facets"]["F1"][0]["metrics"][0][
+                        "value"
+                    ]
+                )
+                <= 1.0
+        )
+        check_analysis_config(data_config, sagemaker_session, "post_training_bias")
+
+
 def check_analysis_config(data_config, sagemaker_session, method):
     analysis_config_json = s3.S3Downloader.read_file(
         data_config.s3_output_path + "/analysis_config.json",
