feat: extend static analysis and compute confidence scores for deploy commands (#673)

This PR extends and makes changes to the static analysis of CI configurations in Macaron with the high-level goal of finding build and deploy commands more accurately. To achieve that, some of the abstractions had to be replaced to allow writing customized analyses, such as detecting build language setup in a GitHub Actions workflow or detecting reachable secrets. Here are the summary of changes:

* BashCommands which consisted of build tool commands collected after analyzing GitHub Actions and passed to checks is replaced with BuildToolCommand.
* The build related checks are refactored and simplified to use BuildToolCommand.
* The callgraph analysis, which needs to be implemented for each CI service, is extended with new node types for GitHub Actions. The callgraph plays the role of Intermediate Representation and is available to all checks.
* The mcn_build_script_1 check does not depend on any checks and always runs by default based on a customer request.
* The mcn_build_as_code_1 check now reports deploy commands with confidence scores.
* New analysis is added to resolve the value of expression variables, which is used for other analysis, such as reachable secrets and build language detection.
* New abstractions are added to model third-party GitHub Actions. This feature is used to collect data about build Language setup

Signed-off-by: behnazh-w <[email protected]>

Author: behnazh-w

docs/source/assets/er-diagram.svg

@@ -41,6 +41,14 @@ macaron.slsa\_analyzer.build\_tool.gradle module
    :undoc-members:
    :show-inheritance:
 
+macaron.slsa\_analyzer.build\_tool.language module
+--------------------------------------------------
+
+.. automodule:: macaron.slsa_analyzer.build_tool.language
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 macaron.slsa\_analyzer.build\_tool.maven module
 -----------------------------------------------
 

docs/source/pages/developers_guide/apidoc/macaron.slsa_analyzer.build_tool.rst

@@ -0,0 +1,26 @@
+macaron.slsa\_analyzer.ci\_service.github\_actions package
+==========================================================
+
+.. automodule:: macaron.slsa_analyzer.ci_service.github_actions
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Submodules
+----------
+
+macaron.slsa\_analyzer.ci\_service.github\_actions.analyzer module
+------------------------------------------------------------------
+
+.. automodule:: macaron.slsa_analyzer.ci_service.github_actions.analyzer
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+macaron.slsa\_analyzer.ci\_service.github\_actions.github\_actions\_ci module
+-----------------------------------------------------------------------------
+
+.. automodule:: macaron.slsa_analyzer.ci_service.github_actions.github_actions_ci
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/pages/developers_guide/apidoc/macaron.slsa_analyzer.ci_service.github_actions.rst

@@ -6,6 +6,14 @@ macaron.slsa\_analyzer.ci\_service package
    :undoc-members:
    :show-inheritance:
 
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 1
+
+   macaron.slsa_analyzer.ci_service.github_actions
+
 Submodules
 ----------
 
@@ -25,14 +33,6 @@ macaron.slsa\_analyzer.ci\_service.circleci module
    :undoc-members:
    :show-inheritance:
 
-macaron.slsa\_analyzer.ci\_service.github\_actions module
----------------------------------------------------------
-
-.. automodule:: macaron.slsa_analyzer.ci_service.github_actions
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 macaron.slsa\_analyzer.ci\_service.gitlab\_ci module
 ----------------------------------------------------
 

docs/source/pages/developers_guide/apidoc/macaron.slsa_analyzer.ci_service.rst

@@ -1,4 +1,4 @@
-/* Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. */
+/* Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. */
 /* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */
 
 // Package bashparser parses the bash scripts and provides parsed objects in JSON.
@@ -21,16 +21,17 @@ type CMDResult struct {
 // ParseCommands parses the bash script to find bash commands.
 // It returns the parsed commands in JSON format.
 func ParseCommands(data string) (string, error) {
-	// Remove GitHub Actions's expressions because the bash parser doesn't recognize it.
-	// We use greedy matching, so if we have `${{ $ {{ foo }} }}`, it will be matched
-	// to `$MACARON_UNKNOWN`, even though it's not a valid GitHub expression.
+	// Replace GitHub Actions's expressions with ``$MACARON_UNKNOWN``` variable because the bash parser
+	// doesn't recognize such expressions. For example: ``${{ foo }}`` will be replaced by ``$MACARON_UNKNOWN``.
+	// Note that we don't use greedy matching, so if we have `${{ ${{ foo }} }}`, it will not be replaced by
+	// `$MACARON_UNKNOWN`.
 	// See: https://docs.github.com/en/actions/learn-github-actions/expressions.
-	var re, reg_error = regexp.Compile(`\$\{\{.*\}\}`)
+	var re, reg_error = regexp.Compile(`\$\{\{.*?\}\}`)
 	if reg_error != nil {
 		return "", reg_error
 	}
 
-	// We replace the GH Actions variables with "UNKNOWN" for now.
+	// We replace the GH Actions variables with "$MACARON_UNKNOWN".
 	data = string(re.ReplaceAll([]byte(data), []byte("$$MACARON_UNKNOWN")))
 	data_str := strings.NewReader(data)
 	data_parsed, parse_err := syntax.NewParser().Parse(data_str, "")

golang/internal/bashparser/bashparser.go

@@ -81,7 +81,7 @@ echo "Run integration tests without configurations"
 echo -e "==================================================================================\n"
 
 echo -e "\n----------------------------------------------------------------------------------"
-echo "micronaut-projects/micronaut-core: Analyzing the repo path and the branch name when automatic dependency resolution is skipped."
+echo "micronaut-projects/micronaut-core: Analyzing the PURL when automatic dependency resolution is skipped."
 echo -e "----------------------------------------------------------------------------------\n"
 JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/micronaut-core/micronaut-core.json
 JSON_RESULT=$WORKSPACE/output/reports/maven/io_micronaut/micronaut-core/micronaut-core.json
@@ -705,6 +705,17 @@ $RUN_POLICY -f $POLICY_FILE -d "$WORKSPACE/output/macaron.db" || log_fail
 check_or_update_expected_output $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail
 check_or_update_expected_output "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail
 
+echo -e "\n----------------------------------------------------------------------------------"
+echo "Run policy CLI with micronaut-core results to test deploy command information."
+echo -e "----------------------------------------------------------------------------------\n"
+RUN_POLICY="macaron verify-policy"
+POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/micronaut-core/test_deploy_info.dl
+POLICY_RESULT=$WORKSPACE/output/policy_report.json
+POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/micronaut-core/test_deploy_info.json
+
+$RUN_POLICY -f $POLICY_FILE -d "$WORKSPACE/output/macaron.db" || log_fail
+check_or_update_expected_output $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail
+
 # Testing the Repo Finder's remote calls.
 # This requires the 'packageurl' Python module
 echo -e "\n----------------------------------------------------------------------------------"

scripts/dev_scripts/integration_tests.sh

@@ -1,11 +1,11 @@
-# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module contains classes to generate build call graphs for the target repository."""
 
 from collections import deque
 from collections.abc import Iterable
-from typing import Generic, TypeVar
+from typing import Any, Generic, TypeVar
 
 Node = TypeVar("Node", bound="BaseNode")
 # The documentation below for `TypeVar` is commented out due to a breaking
@@ -21,9 +21,22 @@
 class BaseNode(Generic[Node]):
     """This is the generic class for call graph nodes."""
 
-    def __init__(self) -> None:
-        """Initialize instance."""
+    def __init__(self, caller: Node | None = None, node_id: str | None = None) -> None:
+        """Initialize instance.
+
+        Parameter
+        ---------
+        caller: Node | None
+            The caller node.
+        node_id: str | None
+            The unique identifier of a node in the callgraph.
+        """
         self.callee: list[Node] = []
+        self.caller: Node | None = caller
+        # Each node can have a model that summarizes certain properties for static analysis.
+        # By default this model is set to None.
+        self.model: Any = None
+        self.node_id = node_id
 
     def add_callee(self, node: Node) -> None:
         """Add a callee to the current node.

src/macaron/code_analyzer/call_graph.py

@@ -30,6 +30,8 @@ timeout = 30
 [bashparser]
 # This is the timeout (in seconds) for the bashparser.
 timeout = 30
+# The maximum allowed recursion depth when scripts call other scripts.
+recursion_depth = 3
 
 [cue_validator]
 # This is the timeout (in seconds) for the cue_validator.
@@ -199,6 +201,7 @@ builder =
     gradle
     gradlew
 build_arg =
+    build
 deploy_arg =
     artifactoryPublish
     publish
@@ -278,7 +281,6 @@ interpreter =
 interpreter_flag =
     -m
 build_arg =
-    install
     build
     setup.py
 deploy_arg =
@@ -342,11 +344,14 @@ package_lock =
 builder =
     npm
     pnpm
-# Build args not defined since npm build is just a plumbing command https://docs.npmjs.com/cli/v6/commands/npm-build
-# and SLSA v1.0 removes the scripted build requirement https://slsa.dev/spec/v1.0/requirements
 build_arg =
+    install
+build_run_arg =
+    build
 deploy_arg =
     publish
+deploy_run_arg =
+    publish
 [builder.npm.ci.deploy]
 github_actions =
     JS-DevTools/npm-publish
@@ -371,17 +376,23 @@ package_lock =
     yarn.lock
 builder =
     yarn
-# Build args not defined for similar reasons to npm
 build_arg =
+    build
+build_run_arg =
+    build
 deploy_arg =
     publish
+deploy_run_arg =
+    publish
 
 # This is the spec for trusted Go build tool usages.
 [builder.go]
 entry_conf =
 build_configs =
     go.mod
     go.sum
+    .goreleaser.yaml
+    .goreleaser.yml
 builder =
     go
 build_arg =
@@ -411,6 +422,17 @@ trusted_builders =
     slsa-framework/slsa-github-generator/.github/workflows/builder_container-based_slsa3.yml
 # The number of days that GitHub Actions persists the workflow run.
 max_workflow_persist = 90
+# The GitHub Actions configuration workflow files for third-party tools.
+third_party_configurations =
+    codeql-analysis.yaml
+    codeql-analysis.yml
+    codeql-config.yaml
+    codeql-config.yml
+    scorecards-analysis.yaml
+    scorecards-analysis.yml
+    dependabot.yaml
+    dependabot.yml
+    renovate.json
 
 # This is the spec for Jenkins CI.
 [ci.jenkins]

src/macaron/config/defaults.ini

@@ -70,3 +70,15 @@ class JsonError(MacaronError):
 
 class InvalidAnalysisTargetError(MacaronError):
     """When a valid Analysis Target cannot be constructed."""
+
+
+class ParseError(MacaronError):
+    """The errors related to parsers."""
+
+
+class CallGraphError(MacaronError):
+    """The errors related to callgraphs."""
+
+
+class GitHubActionsValueError(MacaronError):
+    """The errors related to GitHub Actions value errors."""

src/macaron/errors.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module is a Python wrapper for the compiled actionparser binary.
@@ -13,9 +13,12 @@
 import logging
 import os
 import subprocess  # nosec B404
+from typing import Any
 
 from macaron.config.defaults import defaults
 from macaron.config.global_config import global_config
+from macaron.errors import JsonError, ParseError
+from macaron.json_tools import json_extract
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -34,6 +37,11 @@ def parse(workflow_path: str, macaron_path: str = "") -> dict:
     -------
     dict
         The parsed workflow as a JSON (dict) object.
+
+    Raises
+    ------
+    ParseError
+        When parsing fails with errors.
     """
     if not macaron_path:
         macaron_path = global_config.macaron_path
@@ -56,15 +64,59 @@ def parse(workflow_path: str, macaron_path: str = "") -> dict:
         subprocess.TimeoutExpired,
         FileNotFoundError,
     ) as error:
-        logger.error("Error while parsing GitHub Action workflow %s: %s", workflow_path, error)
-        return {}
+        raise ParseError(f"Error while parsing GitHub Action workflow {workflow_path}") from error
 
     try:
         if result.returncode == 0:
             parsed_obj: dict = json.loads(result.stdout.decode("utf-8"))
             return parsed_obj
-        logger.error("GitHub Actions parser failed: %s", result.stderr)
-        return {}
+        raise ParseError(f"GitHub Actions parser failed: {result.stderr.decode('utf-8')}")
     except json.JSONDecodeError as error:
-        logger.error("Error while loading the parsed Actions workflow: %s", error)
-        return {}
+        raise ParseError("Error while loading the parsed Actions workflow") from error
+
+
+def get_run_step(step: dict[str, Any]) -> str | None:
+    """Get the parsed GitHub Action run step for inlined shell scripts.
+
+    If the run step cannot be validated this function returns None.
+
+    Parameters
+    ----------
+    step: dict[str, Any]
+        The parsed step object.
+
+    Returns
+    -------
+    str | None
+        The inlined run script or None if the run step cannot be validated.
+    """
+    try:
+        return json_extract(step, ["Exec", "Run", "Value"], str)
+    except JsonError as error:
+        logger.debug(error)
+        return None
+
+
+def get_step_input(step: dict[str, Any], key: str) -> str | None:
+    """Get an input value from a GitHub Action step.
+
+    If the input value cannot be found or the step inputs cannot be validated this function
+    returns None.
+
+    Parameters
+    ----------
+    step: dict[str, Any]
+        The parsed step object.
+    key: str
+        The key to be looked up.
+
+    Returns
+    -------
+    str | None
+        The input value or None if it doesn't exist or the parsed object validation fails.
+    """
+    try:
+        return json_extract(step, ["Exec", "Inputs", key, "Value", "Value"], str)
+    except JsonError as error:
+        logger.debug(error)
+        return None