feat: add exclude and include check in ini config (#254)

Signed-off-by: Trong Nhan Mai <[email protected]>

Author: Trong Nhan Mai

scripts/dev_scripts/integration_tests.sh

@@ -577,6 +577,17 @@ fi
 rm -rf "$SOURCE_REPO"
 rm -rf "$TARGET_REPO"
 
+echo -e "\n----------------------------------------------------------------------------------"
+echo "Running the analysis with all checks excluded. This test should return an error code."
+echo -e "----------------------------------------------------------------------------------\n"
+$RUN_MACARON -dp tests/e2e/defaults/exclude_all_checks.ini analyze -rp https://github.com/apache/maven --skip-deps
+
+if [ $? -eq 0 ];
+then
+    echo -e "Expect non-zero status code but got $?."
+    log_fail
+fi
+
 echo -e "\n----------------------------------------------------------------------------------"
 echo "apache/maven: test analyzing without the environment variable GITHUB_TOKEN being set."
 echo -e "----------------------------------------------------------------------------------\n"
@@ -626,12 +637,24 @@ fi
 
 # Testing the CUE provenance expectation verifier.
 echo -e "\n----------------------------------------------------------------------------------"
-echo "Test verifying CUE provenance expectation."
+echo "Test verifying CUE provenance expectation for ossf/scorecard"
+echo -e "----------------------------------------------------------------------------------\n"
+JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.json
+JSON_RESULT=$WORKSPACE/output/reports/github/ossf/scorecard/scorecard.json
+DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/scorecard.ini
+EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue
+$RUN_MACARON -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/[email protected] --skip-deps || log_fail
+
+check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+
+echo -e "\n----------------------------------------------------------------------------------"
+echo "Test verifying CUE provenance expectation for slsa-verifier"
 echo -e "----------------------------------------------------------------------------------\n"
 JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json
 JSON_RESULT=$WORKSPACE/output/reports/github_com/slsa-framework/slsa-verifier/slsa-verifier.json
 EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue
-$RUN_MACARON analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
+DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini
+$RUN_MACARON -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
 
 check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
 
@@ -648,16 +671,15 @@ check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED ||
 
 # Testing the Souffle policy engine.
 echo -e "\n----------------------------------------------------------------------------------"
-echo "Run policy CLI with slsa-verifier results."
+echo "Run policy CLI with scorecard results."
 echo -e "----------------------------------------------------------------------------------\n"
 RUN_POLICY="macaron verify-policy"
-POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/valid/slsa-verifier.dl
+POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/scorecard/scorecard.dl
 POLICY_RESULT=$WORKSPACE/output/policy_report.json
-POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/policy_report.json
+POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/scorecard/scorecard_policy_report.json
 VSA_RESULT=$WORKSPACE/output/vsa.intoto.jsonl
-VSA_PAYLOAD_EXPECTED=$WORKSPACE/tests/vsa/integration/github_slsa-framework_slsa-verifier/vsa_payload.json
+VSA_PAYLOAD_EXPECTED=$WORKSPACE/tests/vsa/integration/github_slsa-framework_scorecard/vsa_payload.json
 
-# Run policy engine on the database and compare results.
 $RUN_POLICY -f $POLICY_FILE -d "$WORKSPACE/output/macaron.db" || log_fail
 check_or_update_expected_output $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail
 check_or_update_expected_output "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail

scripts/dev_scripts/integration_tests_docker.sh

@@ -124,27 +124,38 @@ $RUN_MACARON_SCRIPT analyze -pe $EXPECTATION_DIR -rp https://github.com/urllib3/
 
 python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
 
+echo -e "\n----------------------------------------------------------------------------------"
+echo "Test verifying CUE provenance expectation for ossf/scorecard"
+echo -e "----------------------------------------------------------------------------------\n"
+JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.json
+JSON_RESULT=$WORKSPACE/output/reports/github/ossf/scorecard/scorecard.json
+DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/scorecard.ini
+EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue
+$RUN_MACARON_SCRIPT -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/[email protected] --skip-deps || log_fail
+
+python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
+
 echo -e "\n----------------------------------------------------------------------------------"
 echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped"
 echo "and CUE file is provided as expectation."
 echo -e "----------------------------------------------------------------------------------\n"
 JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json
 JSON_RESULT=$WORKSPACE/output/reports/github_com/slsa-framework/slsa-verifier/slsa-verifier.json
 EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue
-$RUN_MACARON_SCRIPT analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
+DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini
+$RUN_MACARON_SCRIPT -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
 
 python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
 
 echo -e "\n----------------------------------------------------------------------------------"
-echo "Run policy CLI with slsa-verifier results."
+echo "Run policy CLI with scorecard results."
 echo -e "----------------------------------------------------------------------------------\n"
-POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/valid/slsa-verifier.dl
+POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/scorecard/scorecard.dl
 POLICY_RESULT=$WORKSPACE/output/policy_report.json
-POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/policy_report.json
+POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/scorecard/scorecard_policy_report.json
 VSA_RESULT=$WORKSPACE/output/vsa.intoto.jsonl
-VSA_PAYLOAD_EXPECTED=$WORKSPACE/tests/vsa/integration/github_slsa-framework_slsa-verifier/vsa_payload.json
+VSA_PAYLOAD_EXPECTED=$WORKSPACE/tests/vsa/integration/github_slsa-framework_scorecard/vsa_payload.json
 
-# Run policy engine on the database and compare results.
 $RUN_MACARON_SCRIPT verify-policy -f $POLICY_FILE -d "$WORKSPACE/output/macaron.db" || log_fail
 python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail
 python "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail

src/macaron/config/defaults.ini

@@ -479,3 +479,39 @@ hostname = registry.npmjs.org
 # The attestation REST API.
 attestation_endpoint = -/npm/v1/attestations
 request_timeout = 20
+
+# Configuration options for selecting the checks to run.
+# Both the exclude and include are defined as list of strings:
+#   - The exclude list is used to specify the checks that will not run.
+#   - The include list is used to specify the checks that should run. The checks that are not specified by "include" will be excluded.
+# The final list of checks to run will always be "include" minus "exclude".
+# Each element of a list is either the full ID of a check (e.g mcn_build_as_code_1) or a glob pattern
+# (https://docs.python.org/3/library/fnmatch.html) to match multiple check IDs (e.g "mcn_*" will match all default
+# Macaron checks).
+# Examples
+# 1. Exclude 1 check:
+# ```
+# [analysis.checks]
+# exclude = mcn_build_as_code_1
+# include = *
+# ```
+# 2. Exclude multiple checks:
+# ```
+# [analysis.checks]
+# exclude =
+#     mcn_build_as_code_1
+#     mcn_provenance_level_three_1
+# include = *
+# ```
+# 3. Exclude multiple checks that start with `mcn_provenance`:
+# ```
+# [analysis.checks]
+# exclude =
+#     mcn_provenance*
+# include = *
+# ```
+[analysis.checks]
+# By default, we don't exclude any checks.
+exclude =
+# By default, we run all checks available.
+include = *

src/macaron/errors.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module contains error classes for Macaron."""
@@ -54,3 +54,7 @@ class DuplicateError(MacaronError):
 
 class InvalidHTTPResponseError(MacaronError):
     """Happens when the HTTP response is invalid or unexpected."""
+
+
+class CheckRegistryError(MacaronError):
+    """The Check Registry Error class."""

src/macaron/output_reporter/results.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module contains classes that represent the result of the Macaron analysis."""
@@ -142,6 +142,8 @@ def get_dict(self) -> dict:
             "metadata": {
                 "timestamps": datetime.now().isoformat(sep=" ", timespec="seconds"),
                 "has_passing_check": has_passing_check,
+                "run_checks": registry.checks_to_run,
+                "check_tree": registry.check_tree,
             },
             "target": self.context.get_dict() if self.context else {},
             "dependencies": self.get_dep_summary(),

src/macaron/output_reporter/templates/base_template.html

@@ -286,6 +286,13 @@
         .active {
             display: block;
         }
+
+        .check-tree-node::before {
+            content: "\2514";
+            color: black;
+            display: inline-block;
+            margin-right: 6px;
+        }
     </style>
 </head>
 
@@ -683,6 +690,13 @@
                 document.getElementById("check_report_content").classList.toggle("hidden");
             });
 
+            // Add a listener to toggle the display of run checks.
+            let run_check_toggler = document.getElementById("run-checks");
+            run_check_toggler.addEventListener("click", function() {
+                this.classList.toggle("toggler-extend");
+                document.getElementById("run-checks-tree").classList.toggle("hidden");
+            });
+
             // When loaded, expand all CI services.<n> elements
             setExpandState(document.querySelectorAll(".tree-view-nested-list > * > .caret"), true);
             setExpandState(document.querySelectorAll(".tree-view-nested-list > * > * > * > .caret"), true);

src/macaron/output_reporter/templates/macaron.html

@@ -164,6 +164,31 @@
 {% endcall %}
 {% endmacro %}
 
+{#
+    The `check-tree-node` class will add a small symbol before the check ID
+    indicating a non-root node. Therefore, we don't apply it for the checks without
+    parent (i.e the first level checks).
+#}
+{% macro render_run_checks(check_tree, run_checks) %}
+<ul class="tree-view-nested-list">
+    {% for key, next_level in check_tree.items() recursive %}
+    <li>
+        <div
+            class="{% if loop.depth0 != 0 %}check-tree-node{% endif %}"
+            {% if key not in run_checks %}
+            style="text-decoration-line: line-through;"
+            {% endif %}>
+        {{ key }}
+        </div>
+        <ul>
+        {{ loop(next_level.items()) | indent(2 * 4) }}
+        </ul>
+    </li>
+    {% endfor %}
+</ul>
+{% endmacro %}
+
+
 {# -------------------------------------------- #}
 
 {#
@@ -236,6 +261,13 @@
     <div class="table_caption toggler" id="check_report_title">Reports for Macaron checks</div>
     <div id="check_report_content" class="hidden">
     {% endif %}
+        <div class="table_sub_caption toggler" id="run-checks">Run checks</div>
+        <div id="run-checks-tree" class="hidden">
+        {{ render_run_checks(metadata.check_tree, metadata.run_checks) | indent(4, first=true) }}
+        </div>
+
+        <div class="table_sub_caption">Checks report</div>
+
         {{ render_checks_report(target.checks.results) | indent(8) }}
     </div>
 

src/macaron/slsa_analyzer/analyzer.py

@@ -33,7 +33,7 @@
 
 # To load all checks into the registry
 from macaron.slsa_analyzer.checks import *  # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403
-from macaron.slsa_analyzer.checks.check_result import CheckResult, SkippedInfo
+from macaron.slsa_analyzer.checks.check_result import CheckResult
 from macaron.slsa_analyzer.ci_service import CI_SERVICES
 from macaron.slsa_analyzer.database_store import store_analyze_context_to_db
 from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService
@@ -74,6 +74,12 @@ def __init__(self, output_path: str, build_log_path: str) -> None:
             logger.error("Cannot start the analysis. Exiting ...")
             sys.exit(1)
 
+        logger.info(
+            "The following checks are excluded based on the user configuration: %s",
+            [check for check in registry.get_all_checks_mapping() if check not in registry.checks_to_run],
+        )
+        logger.info("The following checks will be run: %s", registry.checks_to_run)
+
         self.output_path = output_path
 
         # Prepare the directory to store all the build logs in the
@@ -881,11 +887,7 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]:
                         )
                     )
 
-        # TODO: Get the list of skipped checks from user configuration
-        skipped_checks: list[SkippedInfo] = []
-
-        results = registry.scan(analyze_ctx, skipped_checks)
-
+        results = registry.scan(analyze_ctx)
         return results