From 3ef0ecd10cd821eadd4cf3134d43b4cb8d91623d Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 19:35:45 +1000 Subject: [PATCH 01/47] test: add new integration test utility Signed-off-by: Nathan Nguyen --- .github/workflows/_build_docker.yaml | 4 + .pre-commit-config.yaml | 12 + docs/source/pages/developers_guide/index.rst | 10 +- pyproject.toml | 1 + scripts/dev_scripts/integration_tests.sh | 3 + .../dev_scripts/integration_tests_docker.sh | 5 + tests/integration/README.md | 219 ++++ tests/integration/run.py | 984 ++++++++++++++++++ 8 files changed, 1236 insertions(+), 2 deletions(-) create mode 100644 tests/integration/README.md create mode 100644 tests/integration/run.py diff --git a/.github/workflows/_build_docker.yaml b/.github/workflows/_build_docker.yaml index 26721f8d9..0ab977d48 100644 --- a/.github/workflows/_build_docker.yaml +++ b/.github/workflows/_build_docker.yaml @@ -59,6 +59,10 @@ jobs: IMAGE_NAME: ghcr.io/oracle/macaron run: make build-docker + - name: Install dependencies for integration test utility + run: | + python3 -m pip install ruamel.yaml cfgv + # Run the integration tests against the built Docker image. - name: Test the Docker image env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0588930ee..d0b391f99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -161,6 +161,18 @@ repos: # - id: pretty-format-toml # args: [--autofix] +- repo: local + hooks: + - id: integration-test-vet + name: validate integration test cases + entry: python3 + args: + - ./tests/integration/run.py + - vet + - ./tests/integration/cases/... + language: system + pass_filenames: false + # On push to the remote, run the unit tests. - repo: local hooks: diff --git a/docs/source/pages/developers_guide/index.rst b/docs/source/pages/developers_guide/index.rst index 211e0911c..94247a35c 100644 --- a/docs/source/pages/developers_guide/index.rst +++ b/docs/source/pages/developers_guide/index.rst @@ -216,10 +216,16 @@ Finally, you need to register your check by adding it to the :mod:`registry modu Test your check ''''''''''''''' -Finally, you can add tests for you check by adding ``tests/slsa_analyzer/checks/test_repo_check.py`` module. Macaron -uses `pytest `_ and `hypothesis `_ for testing. Take a look +Finally, you can add tests for you check. We utilize two types of tests: unit tests, and integration tests. + +For unit tests, you can add a ``tests/slsa_analyzer/checks/test_repo_check.py`` module. Macaron +uses `pytest `_ and `hypothesis `_ for unit testing. Take a look at other tests for inspiration! +For integration tests, please refer to the README file under ``tests/integration`` for +further instructions and have a look at our existing integration test cases if you need +some examples. + .. toctree:: :maxdepth: 1 diff --git a/pyproject.toml b/pyproject.toml index b0e900a6d..34d58df37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,6 +240,7 @@ minversion = "7.0" addopts = """-vv -ra --tb native \ --doctest-modules --doctest-continue-on-failure --doctest-glob '*.rst' \ --cov macaron \ + --ignore tests/integration \ """ # Consider adding --pdb # https://docs.python.org/3/library/doctest.html#option-flags doctest_optionflags = "IGNORE_EXCEPTION_DETAIL" diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index e8c6a6257..1904bd028 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -855,6 +855,9 @@ run_macaron_clean $ANALYZE -purl pkg:maven/io.github.behnazh-w.demo/example-mave check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail +python3 ./tests/integration/run.py run \ + ./tests/integration/cases/... || log_fail + # Important: This should be at the end of the file if [ $RESULT_CODE -ne 0 ]; then diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 9f289b723..148fda15c 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -262,6 +262,11 @@ then log_fail fi +python3 ./tests/integration/run.py run \ + --macaron scripts/release_scripts/run_macaron.sh \ + --include-tag docker \ + ./tests/integration/cases/... || log_fail + if [ $RESULT_CODE -ne 0 ]; then exit 1 diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 000000000..710eb443b --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,219 @@ +# Macaron Integration Tests + + +## Integration Test Utility + +We have an integration test utility script, [`./tests/integration/run.py`](./run.py), for running integration tests. The script should be called within the dev virtual environment and from the root directory of the repository. + +```bash +$ python3 ./tests/integration/run.py -h +usage: ./tests/integration/run.py [-h] {check,vet,run,update} ... + +positional arguments: + {check,vet,run,update} + The command to run. + check Schema-validate test case config files in the test data directory. + vet Validate test case directories. + run Run test cases in the test data directory. + update Run test cases, but update expected output files instead of + comparing them with expected output. + +options: + -h, --help show this help message and exit +``` + +The utility offers 4 different commands, as shown in the help message above. Some example usages for these commands will be introduced later in the [Example Usages section](#example-usages). You can also have a look at the help message of each command by providing ` -h` to the script. + +```bash +# Show help message for the check command. +$ python3 ./tests/integration/run.py check -h +``` + + +## Test Case Configuration + +The utility interacts with integration test cases. Each test case locates in a single directory containing a test config file `test.yaml`, alongside other files required for the test case (e.g. config files, policy files, provenance expectation files). + +Here is an example. The test case directory looks like this. Alongside the `test.yaml` file, there is a file named `analysis_report.json` storing the expected output of an analysis report. + +``` +./tests/integration/cases/apache_maven/ +├── analysis_report.json +└── test.yaml +``` + +Here is the content of the `test.yaml` file. + +```yaml +description: > + Analyze with PURL and repository path without dependency resolution. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - --package-url + - pkg:maven/apache/maven + - --repo-path + - https://github.com/apache/maven + - -b + - master + - -d + - 3fc399318edef0d5ba593723a24fff64291d6f9b + - --skip-deps +- name: Compare analysis report + kind: compare + options: + kind: analysis_report + result: output/reports/maven/apache/maven/maven.json + expected: analysis_report.json +``` + +Each test config file has a description at the top telling what the test case is about, and a sequence of steps to be executed. The execution of a test case stops whenever a step fails (i.e. exits with non-zero code, by default). In the example above, there are 2 steps: (1) run the `macaron analyze` command, and (2) compare a JSON analysis report with the corresponding expected result file. Note that all steps are executed with the test case directory being the current working directory. Therefore, all filepaths in the test config file are relative to the test case directory. + + +## Example Usages + +### Create a new test case + +You create a new test case by creating a new directory, then a `test.yaml` within it. To schema-validate the `test.yaml` file, you can use the `check` command and point to the test case directory: + +```bash +# Schema-validate the ./test/case/directory/test.yaml file. +$ python3 ./tests/integration/run.py check ./test/case/directory +``` + +At this point, some expected result files does not exist yet, since you normally want to run `macaron` once, inspect the result files, then turn them to expected result files if they look good. To do this, you can run in **interactive** mode. In this mode, the utility stops at each steps and ask if you want to run or skip a step. For compare steps, the utility also ask if you want to "update" the expected result file instead of compare. + +```bash +# Run a test case in interactive mode. +$ python3 ./tests/integration/run.py run -i ./test/case/directory +``` + +After you have finished running the test case, you can rerun the test case to make sure everything works as expected. + +```bash +# Run a test case end-to-end. +$ python3 ./tests/integration/run.py run ./test/case/directory +``` + +### Inspect test cases + +Besides the interactive mode, the `run` command also has another special mode called dry-run mode, enabled with the flag `-d/--dry`. In this mode, the utility only shows what commands will be run during the execution of the test cases without actually running any of them. This is especially useful for debugging purposes. + +```bash +# Run a test case in dry-run mode. +$ python3 ./tests/integration/run.py run -d ./test/case/directory +``` + +### Validate test cases before pushing commits to remote or running in CI + +Integration test cases take some reasonable amount of time to run. The `vet` command not only does schema validation on test config files, but also carries out additional validations for each test case to prevent as many unintentional errors as possible and save us time waiting just to see CI failing eventually. + +The `vet` command is meant to be used in CI before running integration test. It is also a useful static check in general. Hence, it has been added as a hook to `pre-commit`. + +### Bulk-process multiple test cases + +All commands (`check`, `vet`, `run`, and `update`) can process multiple test cases, one after another. You can specify more than one directory as positional arguments of these commands. + +```bash +# Run two test cases one after another. +$ python3 ./tests/integration/run.py run ./test_case_a/directory ./test_case_b/directory +``` + +You can also use the `...` path wildcard to allow for discovering test case directories recursively under a root directory. + +```bash +# Run all test cases discovered recursively under a directory. +$ python3 ./tests/integration/run.py run ./all/cases/... +``` + +### Select a subset of test cases to run + +In certain cases, we can utilize the feature of tags to select a subset of test cases to run with the `run` command. + +Each test case can be attached with one or more tags in the yaml configuration. For example, you may find some of our test cases having the `docker` tag as follows. + +```yaml +description: ... +tags: +- docker +steps: +- ... +``` + +We typically have the test cases for the container image being a subset of the test cases for the Macaron Python package. We can mark the test cases shared for both purposes with the `docker` tag. When we do integration testing for the container image, we can add the argument `--include-tag docker` to filter only test cases tagged with `docker`. + +```bash +# Test the container image with test cases having the `docker` tag. +$ python3 ./tests/integration/run.py run --include-tag docker ./all/cases/... +``` + +The `--include-tag` flag can be specified multiple times. A selected test case must contain all tags specified with the `--include-tag` flag. + +```bash +# Test the container image with test cases having the `docker` tag. +$ python3 ./tests/integration/run.py run --include-tag tag-a --include-tag tag-b ./all/cases/... +``` + +There is also the `--exclude-tag` flag. A selected test case must also not contain any tag specified with the `--exclude-tag` flag. + +```bash +# Only run test cases not tagged with `npm`. +$ python3 ./tests/integration/run.py run --exclude-tag npm ./all/cases/... +``` + +You can simply think of each `--include-tag`/`--exclude-tag` argument as adding an additional constraint that a selected test case must satisfy". + +### Debug utility script + +In case you want to debug the utility script itself, there is the verbose mode for all commands which can be enabled with the `-v/--verbose` flag. + + +## Test Config Reference + +### Test case Schema + +* `description` (`string`, required): The description of the test case. +* `tags` (`array[string]`, optional, default is `[]`): The tags of the test case. When the `--tag ` argument is passed with the `run` command , only run test cases having the tag ``. (The `--tag` argument is typically used in combination with test case discovery using the `...` wildcard). +* `steps` (`array[string]`, required): The list of steps in a test case. Steps in a test case are executed sequentially. A test case stops execution and fails if any command fails. + +### Step Schema + +* `name` (`string`, required): The name of the step. +* `kind` (`"analyze" | "verify" | "compare" | "shell"`, required): The kind of the step. There are 4 kinds of steps: + * `"analyze"`: runs the `macaron analyze` command. + * `"verify"`: runs the `macaron verify-policy` command. + * `"compare"`: compares an output file with an expected output file. + * `"shell"`: runs an arbitrary shell command. +* `options`: Configuration options for the step. These options are specific to the step kind. See their schema below. +* `env` (`dict[string, string | null]`, optional): Key value pairs of environment variables being modified during the step after inheriting the environment in which the utility is executed within. Each value can be a string if you want to set a value to the environment variable, or null if you want to "unset" the variable. +* `expect_fail` (`bool`, optional, default is `false`): If `true`, assert that the step must exit with non-zero code. This should be used for cases where we expect a command to fail. + +### Analyze step options Schema + +* `main_args` (`array[string]`, optional): main arguments for `macaron`, i.e. those specified before the `analyze` command, e.g. `--verbose`. +* `command_args` (`array[string]`, optional): arguments for the `analyze` command. +* `ini` (`string`, optional): The `.ini` configuration file (a relative path from test case directory). This enables additional validations and is recommended over passing `--defaults-path` and the config file through `main_args`. +* `expectation` (`string`, optional): The provenance expectation file in CUE) (a relative path from test case directory). This enables additional validations and is recommended over passing `--provenance-expectation` and the expectation file through `command_args`. +* `provenance`: (`string`, optional): The provenance file (a relative path from test case directory). This enables additional validations and is recommended over passing `--provenance-file` and the provenance file through `command_args`. +* `sbom`: (`string`, optional): The SBOM file (a relative path from test case directory). This enables additional validations and is recommended over passing `--sbom-file` and the SBOM file through `command_args`. + +### Verify step options Schema + +* `main_args` (`array[string]`, optional): main arguments for `macaron`, i.e. those specified before the `verify-command` command, e.g. `--verbose`. +* `command_args` (`array[string]`, optional): arguments for the `verify-policy` command. +* `policy` (`string`, optional): The `policy.dl` file. This enables additional validations and is recommended over passing `--file` and the policy file through `command_args`. +* `database` (`string`, optional, default is `./output/macaron.db`): The database file. This is recommended over passing `--database` and the database file through `command_args`. +* `show_prelude` (`bool`, optional, default is `false`): Run the command in `--show-prelude` mode. + +### Compare step options Schema + +* `kind` (`"analysis_report" | "policy_report" | "deps_report" | "vsa"`, required): The kind of JSON report to compare. +* `result` (`string`, required): The output file (a relative path from test case directory). +* `expected` (`string`, required): The expected output file (a relative path from test case directory). + +### Shell step options Schema + +* `cmd` (`string`, required): The shell command to run. diff --git a/tests/integration/run.py b/tests/integration/run.py new file mode 100644 index 000000000..877e6fc0b --- /dev/null +++ b/tests/integration/run.py @@ -0,0 +1,984 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""Integration test utility.""" + +from __future__ import annotations + +import argparse +import glob +import logging +import logging.config +import os +import shutil +import subprocess # nosec B404 +import sys +import time +from abc import abstractmethod +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Generic, TypedDict, TypeVar, cast + +import cfgv # type: ignore +from ruamel.yaml import YAML + +T = TypeVar("T") + +logger = logging.getLogger(sys.argv[0]) + +environ = dict(os.environ) +# Disable pulling the release docker image to test the locally built image instead. +environ["DOCKER_PULL"] = "never" + + +def patch_env(patch: Mapping[str, str | None]) -> dict[str, str]: + """Patch env.""" + copied_env = dict(environ) # Make a copy. + + for var, value in patch.items(): + if value is None: + copied_env.pop(var, None) + else: + copied_env[var] = value + + return copied_env + + +def configure_logging(verbose: bool) -> None: + """Configure logging.""" + logging.config.dictConfig( + { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "standard": { + "format": "[%(levelname)s|%(module)s]: %(message)s", + }, + "verbose": { + "format": "[%(levelname)s|%(module)s|L%(lineno)d]: %(message)s", + }, + }, + "handlers": { + "stderr": { + "class": "logging.StreamHandler", + "level": "DEBUG" if verbose else "INFO", + "formatter": "verbose" if verbose else "standard", + "stream": "ext://sys.stderr", + }, + }, + "root": { + "level": "DEBUG", + "handlers": ["stderr"], + }, + } + ) + + +COMPARE_SCRIPTS: dict[str, Sequence[str]] = { + "analysis_report": ["tests", "e2e", "compare_e2e_result.py"], + "policy_report": ["tests", "policy_engine", "compare_policy_reports.py"], + "deps_report": ["tests", "dependency_analyzer", "compare_dependencies.py"], + "vsa": ["tests", "vsa", "compare_vsa.py"], +} + + +def check_required_file(cwd: str) -> Callable[[str], None]: + """Check for a required file of a test case.""" + + def _check(v: str) -> None: + filepath = os.path.join(cwd, v) + if not os.path.isfile(filepath): + raise cfgv.ValidationError(f"File {filepath} does not exist.") + + return cast(Callable[[str], None], cfgv.check_and(cfgv.check_string, _check)) + + +def check_env(env: object) -> None: + """Check for a required file of a test case.""" + if not isinstance(env, dict): + raise cfgv.ValidationError("Value of the 'env' field must be a dictionary.") + for k, v in env.items(): + if not (v is None or isinstance(v, str)): + raise cfgv.ValidationError(f"Value of key '{k}' is not a str or null.") + + +class StepConfig(TypedDict): + """Configuration for a step.""" + + name: str + kind: str + options: dict + + +@dataclass +class Step(Generic[T]): + """A step in a test case.""" + + step_id: int + name: str + options: T + env: dict[str, str | None] + expect_fail: bool + + @abstractmethod + def cmd(self, macaron_cmd: str) -> list[str]: + """Get the shell command of the step.""" + raise NotImplementedError() + + def show_command(self, macaron_cmd: str) -> None: + """Log the command that the step runs.""" + args = self.cmd(macaron_cmd=macaron_cmd) + logger.info("Step [%s] '%s'", self.step_id, self.name) + logger.info("Command: '%s'", " ".join(args)) + + def run_command(self, cwd: str, macaron_cmd: str) -> int: + """Run the step.""" + args = self.cmd(macaron_cmd=macaron_cmd) + logger.info("Start running step [%s] '%s'", self.step_id, self.name) + logger.info("Command: '%s'", " ".join(args)) + + start_time = time.monotonic_ns() + res = subprocess.run( + args=args, + cwd=cwd, + env=patch_env(self.env), + check=False, + ) # nosec: B603 + end_time = time.monotonic_ns() + + if self.expect_fail: + if res.returncode == 0: + logger.error( + "Command '%s' unexpectedly exited with zero code while non-zero code expected.", + " ".join(args), + ) + return 1 + else: + if res.returncode != 0: + logger.error( + "Command '%s' unexpectedly exited with non-zero code.", + " ".join(args), + ) + return 1 + + time_taken = (end_time - start_time) / 1e9 + logger.info( + "Time taken for step [%s] '%s': %.4f seconds.", + *(self.step_id, self.name, time_taken), + ) + return 0 + + def run_interactively(self, cwd: str, macaron_cmd: str) -> int: + """Run in interactive mode.""" + inp = None + while inp not in ["y", "n"]: + inp = input(f"> Run step [{self.step_id}] '{self.name}' ('y' for yes/'n' for no)? ") + if inp == "y": + return self.run_command(cwd=cwd, macaron_cmd=macaron_cmd) + return 0 + + +class ShellStepOptions(TypedDict): + """The configuration options of a shell step.""" + + cmd: str + + +@dataclass +class ShellStep(Step[ShellStepOptions]): + """A shell step in a test case, which allows for running arbitrary shell commands.""" + + @staticmethod + def options_schema() -> cfgv.Map: + """Generate the schema of a shell step.""" + return cfgv.Map( + "shell options", + None, + *[ + cfgv.Required(key="cmd", check_fn=cfgv.check_string), + ], + ) + + def cmd(self, macaron_cmd: str) -> list[str]: + return self.options["cmd"].strip().split() + + +class CompareStepOptions(TypedDict): + """Configuration of a compare step.""" + + kind: str + result: str + expected: str + + +@dataclass +class CompareStep(Step[CompareStepOptions]): + """A compare step.""" + + @staticmethod + def options_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map: + """Generate the schema of a compare step.""" + if check_expected_result_files: + check_file = check_required_file(cwd) + else: + check_file = cfgv.check_string + + return cfgv.Map( + "compare options", + None, + *[ + cfgv.Required( + key="kind", + check_fn=cfgv.check_one_of(tuple(COMPARE_SCRIPTS.keys())), + ), + cfgv.Required( + key="result", + check_fn=cfgv.check_string, + ), + cfgv.Required( + key="expected", + check_fn=check_file, + ), + ], + ) + + def cmd(self, macaron_cmd: str) -> list[str]: + kind = self.options["kind"] + result_file = self.options["result"] + expected_file = self.options["expected"] + return [ + "python3", + os.path.abspath(os.path.join(*COMPARE_SCRIPTS[kind])), + *[result_file, expected_file], + ] + + def run_interactively(self, cwd: str, macaron_cmd: str) -> int: + """Run in interactive mode.""" + inp = None + while inp not in ["y", "n", "u"]: + inp = input(f"> Run step {[self.step_id]} '{self.name}' ('y' for yes/'n' for no/'u' for update)? ") + if inp == "y": + return self.run_command(cwd=cwd, macaron_cmd=macaron_cmd) + if inp == "u": + return self.update_result(cwd=cwd) + return 0 + + def update_result(self, cwd: str) -> int: + """Update the expected result file.""" + kind = self.options["kind"] + result_file = os.path.join(cwd, self.options["result"]) + expected_file = os.path.join(cwd, self.options["expected"]) + if kind == "vsa": + proc = subprocess.run( + args=[ + "python3", + os.path.abspath(os.path.join(*COMPARE_SCRIPTS[kind])), + "--update", + *[result_file, expected_file], + ], + check=False, + ) # nosec: B603 + if proc.returncode != 0: + logger.error("Failed to update %s.", expected_file) + return 1 + else: + try: + shutil.copy2(result_file, expected_file) + except OSError as err: + logger.error( + "Failed to copy %s to %s: %s", + *(result_file, expected_file, err), + ) + return 1 + + logger.info( + "Updated %s %s from %s successfully.", + *(kind, expected_file, result_file), + ) + return 0 + + +class AnalyzeStepOptions(TypedDict): + """The configuration options of an analyze step.""" + + main_args: Sequence[str] + command_args: Sequence[str] + ini: str | None + expectation: str | None + provenance: str | None + sbom: str | None + + +@dataclass +class AnalyzeStep(Step): + """A step running the ``macaron analyze`` command.""" + + @staticmethod + def options_schema(cwd: str) -> cfgv.Map: + """Generate the schema of an analyze step.""" + return cfgv.Map( + "analyze options", + None, + *[ + cfgv.NoAdditionalKeys( + [ + "main_args", + "command_args", + "env", + "ini", + "expectation", + "provenance", + "sbom", + ], + ), + cfgv.Optional( + key="main_args", + check_fn=cfgv.check_array(cfgv.check_string), + default=[], + ), + cfgv.Optional( + key="command_args", + check_fn=cfgv.check_array(cfgv.check_string), + default=[], + ), + cfgv.Optional( + key="ini", + check_fn=check_required_file(cwd), + default=None, + ), + cfgv.Optional( + key="expectation", + check_fn=check_required_file(cwd), + default=None, + ), + cfgv.Optional( + key="provenance", + check_fn=check_required_file(cwd), + default=None, + ), + cfgv.Optional( + key="sbom", + check_fn=check_required_file(cwd), + default=None, + ), + ], + ) + + def cmd(self, macaron_cmd: str) -> list[str]: + """Generate the command of the step.""" + args = [macaron_cmd] + args.extend(self.options["main_args"]) + ini_file = self.options.get("ini", None) + if ini_file is not None: + args.extend(["--defaults-path", ini_file]) + args.append("analyze") + expectation_file = self.options.get("expectation", None) + if expectation_file is not None: + args.extend(["--provenance-expectation", expectation_file]) + provenance_file = self.options.get("provenance", None) + if provenance_file is not None: + args.extend(["--provenance-file", provenance_file]) + sbom_file = self.options.get("sbom", None) + if sbom_file is not None: + args.extend(["--sbom-path", sbom_file]) + args.extend(self.options["command_args"]) + return args + + +class VerifyStepOptions(TypedDict): + """The configuration options of a verify step.""" + + main_args: Sequence[str] + command_args: Sequence[str] + database: str + policy: str | None + show_prelude: bool + + +@dataclass +class VerifyStep(Step[VerifyStepOptions]): + """A step running the ``macaron verify-policy`` command.""" + + @staticmethod + def options_schema(cwd: str) -> cfgv.Map: + """Generate the schema of a verify step.""" + return cfgv.Map( + "verify options", + None, + *[ + cfgv.Optional( + key="main_args", + check_fn=cfgv.check_array(cfgv.check_string), + default=[], + ), + cfgv.Optional( + key="command_args", + check_fn=cfgv.check_array(cfgv.check_string), + default=[], + ), + cfgv.Optional( + key="database", + check_fn=cfgv.check_string, + default="./output/macaron.db", + ), + cfgv.Optional( + key="policy", + check_fn=check_required_file(cwd), + default=None, + ), + cfgv.Optional( + key="show_prelude", + check_fn=cfgv.check_bool, + default=False, + ), + ], + ) + + def cmd(self, macaron_cmd: str) -> list[str]: + """Generate the command of the step.""" + args = [macaron_cmd] + args.extend(self.options["main_args"]) + args.append("verify-policy") + args.extend(["--database", self.options["database"]]) + args.extend(self.options["command_args"]) + policy_file = self.options["policy"] + if policy_file is not None: + args.extend(["--file", policy_file]) + show_prelude = self.options["show_prelude"] + if show_prelude: + args.extend(["--show-prelude"]) + return args + + +def gen_step_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map: + """Generate schema for a step.""" + return cfgv.Map( + "steps[*]", + "name", + *[ + cfgv.Required( + key="name", + check_fn=cfgv.check_string, + ), + cfgv.Required( + key="kind", + check_fn=cfgv.check_one_of( + ( + "shell", + "compare", + "analyze", + "verify", + ), + ), + ), + cfgv.ConditionalRecurse( + condition_key="kind", + condition_value="command", + key="options", + schema=ShellStep.options_schema(), + ), + cfgv.ConditionalRecurse( + condition_key="kind", + condition_value="compare", + key="options", + schema=CompareStep.options_schema( + cwd=cwd, + check_expected_result_files=check_expected_result_files, + ), + ), + cfgv.ConditionalRecurse( + condition_key="kind", + condition_value="analyze", + key="options", + schema=AnalyzeStep.options_schema(cwd=cwd), + ), + cfgv.ConditionalRecurse( + condition_key="kind", + condition_value="verify", + key="options", + schema=VerifyStep.options_schema(cwd=cwd), + ), + cfgv.Optional( + key="env", + check_fn=check_env, + default={}, + ), + cfgv.Optional( + key="expect_fail", + check_fn=cfgv.check_bool, + default=False, + ), + ], + ) + + +class CaseConfig(TypedDict): + """The configuration of a test case.""" + + description: str + tags: Sequence[str] + steps: Sequence[StepConfig] + + +@dataclass +class Case: + """A single test case.""" + + case_dir: str + description: str + steps: list[Step] + + def run(self, macaron_cmd: str, interactive: bool, dry: bool) -> int: + """Run the case.""" + logger.info("-" * 60) + logger.info("Case started: '%s'.", self.case_dir) + for line in self.description.strip().splitlines(): + logger.info(" %s", line) + + ret = 0 + + # Clean up previous results + output_dir = os.path.join(self.case_dir, "output") + if not dry and os.path.isdir(output_dir): + remove_output = True + if interactive: + inp = None + while inp not in ["y", "n"]: + inp = input(f"> Remove {output_dir}? (y/n) ") + if inp == "n": + remove_output = False + if remove_output: + logger.info("Removing old copy of '%s'", output_dir) + shutil.rmtree(output_dir) + + for step in self.steps: + if dry: + step.show_command(macaron_cmd=macaron_cmd) + elif interactive: + ret = step.run_interactively(cwd=self.case_dir, macaron_cmd=macaron_cmd) + else: + ret = step.run_command(cwd=self.case_dir, macaron_cmd=macaron_cmd) + if ret != 0: + logger.error("Case failed: '%s'.", self.case_dir) + return ret + + if not dry: + logger.info("Case passed: '%s'.", self.case_dir) + return 0 + + def update(self, macaron_cmd: str) -> int: + """Run the test case in update mode.""" + for step in self.steps: + if isinstance(step, CompareStep): + ret = step.update_result(cwd=self.case_dir) + else: + ret = step.run_command(cwd=self.case_dir, macaron_cmd=macaron_cmd) + + if ret != 0: + return ret + + return 0 + + @staticmethod + def schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map: + """Generate the schema of a compare step.""" + return cfgv.Map( + "test_case", + None, + *[ + cfgv.Required( + key="description", + check_fn=cfgv.check_string, + ), + cfgv.Optional( + key="tags", + check_fn=cfgv.check_array(cfgv.check_string), + default=[], + ), + cfgv.RequiredRecurse( + key="steps", + schema=cfgv.Array( + of=gen_step_schema( + cwd=cwd, + check_expected_result_files=check_expected_result_files, + ), + allow_empty=False, + ), + ), + ], + ) + + +def find_test_case_dirs_under(root_dir: str) -> set[str]: + """Find all test case directories under a root directory. + + Each directory containing a ``test.yaml`` file is a test case directory. + """ + test_case_dirs = set() + for test_config_file in glob.iglob(f"{root_dir}/**/test.yaml", recursive=True): + test_case_dir = os.path.dirname(test_config_file) + test_case_dirs.add(test_case_dir) + return test_case_dirs + + +def collect_test_case_dirs(test_case_dir_args: list[str]) -> list[str]: + """Collect all test case directories given a list of CLI input arguments. + + Parameters + ---------- + test_case_dir_args : list[str] + Each test case directory argument should either: + - Be a directory containing a ``test.yaml`` file, or + - Be a glob ending with ``/...``, which triggers recursively searching + subdirectories for those containing a ``test.yaml`` file. + + Returns + ------- + list[str] + Test case directories, each contains a ``test.yaml`` file. + """ + test_case_dirs: set[str] = set() + for test_case_dir in test_case_dir_args: + if test_case_dir.endswith("/..."): + test_case_dirs = test_case_dirs.union(find_test_case_dirs_under(root_dir=test_case_dir[:-4])) + else: + test_case_dirs.add(test_case_dir) + return sorted(test_case_dirs) + + +class InvalidConfigError(cfgv.ValidationError): + """Error raised when there is a schema error in a test config file.""" + + +def load_config( + test_config_dir: str, + check_expected_result_files: bool, +) -> CaseConfig: + """Test a ``test.yaml`` file.""" + test_config_file = os.path.join(test_config_dir, "test.yaml") + yaml = YAML(typ="safe") + return cast( + CaseConfig, + cfgv.load_from_filename( + filename=test_config_file, + schema=Case.schema( + check_expected_result_files=check_expected_result_files, + cwd=test_config_dir, + ), + load_strategy=yaml.load, + exc_tp=InvalidConfigError, + ), + ) + + +def parse_step_config(step_id: int, step_config: Mapping) -> Step: + """Parse the configuration of a step.""" + kind = step_config["kind"] + step_cls = { + "analyze": AnalyzeStep, + "verify": VerifyStep, + "shell": ShellStep, + "compare": CompareStep, + }[kind] + return step_cls( # type: ignore # https://github.com/python/mypy/issues/3115 + step_id=step_id, + name=step_config["name"], + options=step_config["options"], + env=step_config["env"], + expect_fail=step_config["expect_fail"], + ) + + +def load_test_cases( + test_case_dirs: list[str], + check_expected_result_files: bool, + include_tags: list[str] | None = None, + exclude_tags: list[str] | None = None, +) -> list[Case] | None: + """Load the test cases from the test case directories. + + Parameters + ---------- + test_case_dirs : list[str] + Test case directores. + check_expected_result_files : bool + Whether to check if expected result files are valid. + include_tags : list[str] | None + A selected test case must contain all of these tags. + exclude_tags : list[str] | None + A selected test case must not contain any of these tags. + """ + include_tags = include_tags or [] + exclude_tags = exclude_tags or [] + + err = False + test_cases: list[Case] = [] + + for test_case_dir in test_case_dirs: + try: + case_config = load_config(test_case_dir, check_expected_result_files) + except InvalidConfigError as exc: + logger.error("Case '%s' fails validation.", test_case_dir) + logger.error(exc.error_msg) + err = True + else: + # Each --include-tag/--exclude-tag argument adds an additional constraint + # that a selected test case needs to satisfy, i.e. a selected test case must: + # - contains all tags specified with --include-tag + # - contains no tag specified with --exclude-tag + select_case = True + for include_tag in include_tags: + if include_tag not in case_config["tags"]: + logger.info( + "Skipping case '%s' for not having tag '%s'.", + *(test_case_dir, include_tag), + ) + select_case = False + break + for exclude_tag in exclude_tags: + if exclude_tag in case_config["tags"]: + logger.info( + "Skipping case '%s' for having tag '%s'", + *(test_case_dir, exclude_tag), + ) + select_case = False + break + if not select_case: + continue + steps = [] + for step_id, step_config in enumerate(case_config["steps"]): + step = parse_step_config(step_id, step_config) + steps.append(step) + test_case = Case( + case_dir=test_case_dir, + description=case_config["description"], + steps=steps, + ) + test_cases.append(test_case) + logger.info("Case '%s' passes validation.", test_case_dir) + + if err: + return None + + return test_cases + + +def do_check(test_case_dirs: list[str], check_expected_result_files: bool) -> int: + """Execute the check command.""" + test_cases = load_test_cases( + test_case_dirs, + check_expected_result_files=check_expected_result_files, + ) + if test_cases is None: + return 1 + return 0 + + +def do_run( + test_case_dirs: list[str], + macaron_cmd: str, + include_tags: list[str], + exclude_tags: list[str], + interactive: bool, + dry: bool, +) -> int: + """Execute the run command.""" + test_cases = load_test_cases( + test_case_dirs, + check_expected_result_files=not interactive, + include_tags=include_tags, + exclude_tags=exclude_tags, + ) + + if test_cases is None: + logger.error("Error encountered while loading test config.") + return 1 + + logger.info("Running the following test cases:") + for test_case in test_cases: + logger.info("* %s", test_case.case_dir) + + for test_case in test_cases: + ret = test_case.run( + macaron_cmd=macaron_cmd, + interactive=interactive, + dry=dry, + ) + if ret != 0: + return ret + return 0 + + +def do_update( + test_case_dirs: list[str], + macaron_cmd: str, + include_tags: list[str], + exclude_tags: list[str], +) -> int: + """Execute the update command.""" + test_cases = load_test_cases( + test_case_dirs, + check_expected_result_files=False, + include_tags=include_tags, + exclude_tags=exclude_tags, + ) + if test_cases is None: + return 1 + + ret = 0 + for test_case in test_cases: + ret = test_case.update(macaron_cmd) + if ret != 0: + return ret + return ret + + +def main(argv: Sequence[str] | None = None) -> int: + """Run main logic.""" + arg_parser = argparse.ArgumentParser(sys.argv[0]) + + shared_arguments_parser = argparse.ArgumentParser(add_help=False) + shared_arguments_parser.add_argument( + "test_case_dirs", + help="Test data directories. Use the `...` wildcard to discover test case directories recursively.", + nargs="+", + ) + shared_arguments_parser.add_argument( + *("-v", "--verbose"), + help="Enable verbose logging", + action="store_true", + default=False, + ) + + command_parsers = arg_parser.add_subparsers( + dest="command", + required=True, + help="The command to run.", + ) + + command_parsers.add_parser( + name="check", + parents=[shared_arguments_parser], + help="Schema-validate test case config files in the test case directories.", + ) + + command_parsers.add_parser( + name="vet", + parents=[shared_arguments_parser], + help="Validate test case directories.", + ) + + run_parser = command_parsers.add_parser( + name="run", + parents=[shared_arguments_parser], + help="Run test cases in the test data directory.", + ) + run_parser.add_argument( + *("-t", "--include-tag"), + help=( + "Select only test cases having the tag. " + "This can be specified multiple times, which will select only cases that have all include tags." + ), + action="append", + default=[], + ) + run_parser.add_argument( + *("-e", "--exclude-tag"), + help=( + "Select only test cases not having the tag. " + "This can be specified multiple times, which will select only cases that do not have any exclude tags." + ), + action="append", + default=[], + ) + run_parser.add_argument( + *("-m", "--macaron"), + help="The command to run Macaron. Note: can be path to the run_macaron.sh script.", + default="macaron", + ) + run_mode_group = run_parser.add_argument_group( + title="Run mode", + description="Special run modes", + ).add_mutually_exclusive_group() + run_mode_group.add_argument( + *("-i", "--interactive"), + action="store_true", + help="Run the test cases in interactive mode.", + ) + run_mode_group.add_argument( + *("-d", "--dry"), + action="store_true", + help=( + "Run the test cases in dry mode, which does not run any command " + "but only shows the commands running during a test case." + ), + ) + + update_parser = command_parsers.add_parser( + name="update", + parents=[shared_arguments_parser], + help="Run test cases, but update expected output files instead of comparing them with expected output.", + ) + update_parser.add_argument( + *("-m", "--macaron"), + help="The command to run Macaron. Note: can be path to the run_macaron.sh script.", + default="macaron", + ) + + args = arg_parser.parse_args(argv) + configure_logging(args.verbose) + + test_case_dirs = collect_test_case_dirs(args.test_case_dirs) + + logger.info("Discovered the following test cases:") + for test_case_dir in test_case_dirs: + logger.info("* %s", test_case_dir) + + if args.command == "check": + return do_check( + test_case_dirs=test_case_dirs, + check_expected_result_files=False, + ) + if args.command == "vet": + return do_check( + test_case_dirs=test_case_dirs, + check_expected_result_files=True, + ) + + for script_key, script_relpath in COMPARE_SCRIPTS.items(): + script_path = os.path.join(".", *script_relpath) + if not os.path.isfile(script_path): + logger.error( + "Compare script for '%s' does not exist at '%s'.", + *(script_key, script_path), + ) + return 1 + + path = shutil.which(args.macaron) + if path is None: + logger.error("'%s' is not a command.") + return 1 + macaron_cmd = os.path.abspath(path) + + if args.command == "run": + return do_run( + test_case_dirs=test_case_dirs, + macaron_cmd=macaron_cmd, + include_tags=args.include_tag, + exclude_tags=args.exclude_tag, + interactive=args.interactive, + dry=args.dry, + ) + if args.command == "update": + return do_update( + test_case_dirs=test_case_dirs, + macaron_cmd=macaron_cmd, + include_tags=args.include_tag, + exclude_tags=args.exclude_tag, + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 63829f3c6481f007d68a91632f92b3fe02fc4143 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 21 Jun 2024 11:07:11 +1000 Subject: [PATCH 02/47] chore: unify behavior when test case loading fails Signed-off-by: Nathan Nguyen --- tests/integration/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/run.py b/tests/integration/run.py index 877e6fc0b..b0d7de489 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -758,6 +758,7 @@ def load_test_cases( logger.info("Case '%s' passes validation.", test_case_dir) if err: + logger.error("Error encountered while loading test config.") return None return test_cases @@ -791,7 +792,6 @@ def do_run( ) if test_cases is None: - logger.error("Error encountered while loading test config.") return 1 logger.info("Running the following test cases:") From 3662a504fc57a448b1cc70beec676e809e6ea71f Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 21 Jun 2024 11:12:37 +1000 Subject: [PATCH 03/47] chore: fix docstring of the `load_config` function Signed-off-by: Nathan Nguyen --- tests/integration/run.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/integration/run.py b/tests/integration/run.py index b0d7de489..643bd5e23 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -654,7 +654,20 @@ def load_config( test_config_dir: str, check_expected_result_files: bool, ) -> CaseConfig: - """Test a ``test.yaml`` file.""" + """Load a test case config. + + Parameters + ---------- + test_config_dir : str + The test case directory containing a ``test.yaml`` file. + check_expected_result_files : bool + Whether to check for expected result files. + + Returns + ------- + CaseConfig + The configuration of the test case. + """ test_config_file = os.path.join(test_config_dir, "test.yaml") yaml = YAML(typ="safe") return cast( From a9fb833b679adc0d7ff797622561375ea451c692 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 21 Jun 2024 11:24:25 +1000 Subject: [PATCH 04/47] chore: update README to remove references to the outdated `--tag` argument Signed-off-by: Nathan Nguyen --- tests/integration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index 710eb443b..c02d03579 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -176,7 +176,7 @@ In case you want to debug the utility script itself, there is the verbose mode f ### Test case Schema * `description` (`string`, required): The description of the test case. -* `tags` (`array[string]`, optional, default is `[]`): The tags of the test case. When the `--tag ` argument is passed with the `run` command , only run test cases having the tag ``. (The `--tag` argument is typically used in combination with test case discovery using the `...` wildcard). +* `tags` (`array[string]`, optional, default is `[]`): The tags of the test case. When the `--include-tag ` and/or `--exclude-tag ` arguments are passed with the `run` command, only run test cases having all `--include-tag` tags and no `--exclude-tag` tags. You can think of it as each `--include-tag`/`--exclude-tag` adds another constraint that a test needs to satisfy for it to be included in a run. These arguments are typically used in combination with test case discovery using the `...` wildcard. * `steps` (`array[string]`, required): The list of steps in a test case. Steps in a test case are executed sequentially. A test case stops execution and fails if any command fails. ### Step Schema From 0a816d8dd918b77837b5c4e48cde4f689a37b630 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 21 Jun 2024 11:26:05 +1000 Subject: [PATCH 05/47] chore: fix some typos and grammatical errors in README Signed-off-by: Nathan Nguyen --- tests/integration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index c02d03579..c4da3a418 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -84,7 +84,7 @@ You create a new test case by creating a new directory, then a `test.yaml` withi $ python3 ./tests/integration/run.py check ./test/case/directory ``` -At this point, some expected result files does not exist yet, since you normally want to run `macaron` once, inspect the result files, then turn them to expected result files if they look good. To do this, you can run in **interactive** mode. In this mode, the utility stops at each steps and ask if you want to run or skip a step. For compare steps, the utility also ask if you want to "update" the expected result file instead of compare. +At this point, some expected result files do not exist yet, since you normally want to run `macaron` once, inspect the result files, then turn them into expected result files if they look good enough. To do this, you can run in **interactive** mode. In this mode, the utility stops at each step and asks if you want to run or skip a step. For `compare` steps, the utility also asks if you want to "update" the expected result file instead of compare. ```bash # Run a test case in interactive mode. From 04815dcdb782165e4912be6d7cd00687da348786 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 21 Jun 2024 11:31:49 +1000 Subject: [PATCH 06/47] chore: fix yaml type of the `steps` array in README Signed-off-by: Nathan Nguyen --- tests/integration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index c4da3a418..745d1ff6f 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -177,7 +177,7 @@ In case you want to debug the utility script itself, there is the verbose mode f * `description` (`string`, required): The description of the test case. * `tags` (`array[string]`, optional, default is `[]`): The tags of the test case. When the `--include-tag ` and/or `--exclude-tag ` arguments are passed with the `run` command, only run test cases having all `--include-tag` tags and no `--exclude-tag` tags. You can think of it as each `--include-tag`/`--exclude-tag` adds another constraint that a test needs to satisfy for it to be included in a run. These arguments are typically used in combination with test case discovery using the `...` wildcard. -* `steps` (`array[string]`, required): The list of steps in a test case. Steps in a test case are executed sequentially. A test case stops execution and fails if any command fails. +* `steps` (`array[Step]`, required; see the Step Schema below): The list of steps in a test case. Steps in a test case are executed sequentially. A test case stops execution and fails if any command fails. ### Step Schema From 46634c86ae3168ef7f997532bc0da0e5311bee44 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 21 Jun 2024 11:34:11 +1000 Subject: [PATCH 07/47] chore: fix step kind name for shell step in cfgv validator Signed-off-by: Nathan Nguyen --- tests/integration/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/run.py b/tests/integration/run.py index 643bd5e23..5bf992789 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -473,7 +473,7 @@ def gen_step_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map: ), cfgv.ConditionalRecurse( condition_key="kind", - condition_value="command", + condition_value="shell", key="options", schema=ShellStep.options_schema(), ), From a11d365a6de339b78d91392be708900d87b9f1d4 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 21 Jun 2024 11:37:30 +1000 Subject: [PATCH 08/47] chore: fix docstring for the `check_env` validator function Signed-off-by: Nathan Nguyen --- tests/integration/run.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/run.py b/tests/integration/run.py index 5bf992789..575c714eb 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -94,12 +94,17 @@ def _check(v: str) -> None: def check_env(env: object) -> None: - """Check for a required file of a test case.""" + """Validate the value of ``env`` in a step. + + The value should be a dict. + Each key should be a string. + Each value should be a string or null. + """ if not isinstance(env, dict): raise cfgv.ValidationError("Value of the 'env' field must be a dictionary.") for k, v in env.items(): if not (v is None or isinstance(v, str)): - raise cfgv.ValidationError(f"Value of key '{k}' is not a str or null.") + raise cfgv.ValidationError(f"Value of key '{k}' is not a string or null.") class StepConfig(TypedDict): From b328f2edfbe42f99b39b6aebc062459755b99a34 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Fri, 21 Jun 2024 20:43:21 +1000 Subject: [PATCH 09/47] chore: adjust script to not stop when a test case fails Signed-off-by: Nathan Nguyen --- tests/integration/run.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/integration/run.py b/tests/integration/run.py index 575c714eb..986479567 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -816,15 +816,35 @@ def do_run( for test_case in test_cases: logger.info("* %s", test_case.case_dir) + failed_case_dirs = set() + exit_code = 0 + for test_case in test_cases: - ret = test_case.run( + case_exit = test_case.run( macaron_cmd=macaron_cmd, interactive=interactive, dry=dry, ) - if ret != 0: - return ret - return 0 + if case_exit != 0: + # Do not exit here, but let all test cases run and aggregate the result. + exit_code = 1 + failed_case_dirs.add(test_case.case_dir) + + if len(test_cases) > 1: + # Only shows this if runs more than one case. + logger.info("=" * 60) + all_cases = {test_case.case_dir for test_case in test_cases} + passed_case_dirs = all_cases.difference(failed_case_dirs) + + logger.info("Number of passed cases: %d/%d.", len(passed_case_dirs), len(test_cases)) + for test_case_dir in passed_case_dirs: + logger.info("* %s", test_case_dir) + + logger.info("Number of failed cases: %d/%d.", len(failed_case_dirs), len(test_cases)) + for test_case_dir in failed_case_dirs: + logger.info("* %s", test_case_dir) + + return exit_code def do_update( From da786a0d87b7ea80636a35db816f374ebc1a2845 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 20:13:48 +1000 Subject: [PATCH 10/47] chore: convert case urllib3_expectation_file Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 10 ------- .../dev_scripts/integration_tests_docker.sh | 11 -------- .../urllib3_expectation_file/expectation.cue | 11 ++++++++ .../cases/urllib3_expectation_file/policy.dl | 22 +++++++++++++++ .../cases/urllib3_expectation_file/test.yaml | 27 +++++++++++++++++++ 5 files changed, 60 insertions(+), 21 deletions(-) create mode 100644 tests/integration/cases/urllib3_expectation_file/expectation.cue create mode 100644 tests/integration/cases/urllib3_expectation_file/policy.dl create mode 100644 tests/integration/cases/urllib3_expectation_file/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 1904bd028..4b8a392a7 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -129,16 +129,6 @@ run_macaron_clean $ANALYZE -rp https://github.com/jenkinsci/plot-plugin -b maste $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped." -echo "The CUE expectation file is provided as a single file path." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl -EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/urllib3_PASS.cue -run_macaron_clean $ANALYZE -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped." echo "The CUE expectation file should be found via the directory path." diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 148fda15c..701d16aca 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -137,17 +137,6 @@ python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail # Clean up and remove the virtual environment. rm -rf "$VIRTUAL_ENV_PATH" -echo -e "\n----------------------------------------------------------------------------------" -echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped." -echo "The CUE expectation file is provided as a single file path." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl -EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/urllib3_PASS.cue - -run_macaron_clean analyze -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped." echo "The CUE expectation file should be found via the directory path." diff --git a/tests/integration/cases/urllib3_expectation_file/expectation.cue b/tests/integration/cases/urllib3_expectation_file/expectation.cue new file mode 100644 index 000000000..8f42a812e --- /dev/null +++ b/tests/integration/cases/urllib3_expectation_file/expectation.cue @@ -0,0 +1,11 @@ +{ + target: "pkg:github.com/urllib3/urllib3", + predicate: { + invocation: { + configSource: { + uri: =~"^git\\+https://github.com/urllib3/urllib3@refs/tags/v?[0-9]+.[0-9]+.[0-9a-z]+$" + entryPoint: ".github/workflows/publish.yml" + } + } + } +} diff --git a/tests/integration/cases/urllib3_expectation_file/policy.dl b/tests/integration/cases/urllib3_expectation_file/policy.dl new file mode 100644 index 000000000..141b722fa --- /dev/null +++ b/tests/integration/cases/urllib3_expectation_file/policy.dl @@ -0,0 +1,22 @@ +/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_build_as_code_1"), + check_passed(component_id, "mcn_build_script_1"), + check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_provenance_available_1"), + check_passed(component_id, "mcn_provenance_expectation_1"), + check_passed(component_id, "mcn_provenance_level_three_1"), + check_passed(component_id, "mcn_version_control_system_1"), + check_failed(component_id, "mcn_infer_artifact_pipeline_1"), + check_failed(component_id, "mcn_provenance_derived_commit_1"), + check_failed(component_id, "mcn_provenance_derived_repo_1"), + check_failed(component_id, "mcn_provenance_witness_level_one_1"), + check_failed(component_id, "mcn_trusted_builder_level_three_1"), + is_repo_url(component_id, "https://github.com/urllib3/urllib3"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:github.com/urllib3/urllib3@87a0ecee6e691fe5ff93cd000c0158deebef763b"). diff --git a/tests/integration/cases/urllib3_expectation_file/test.yaml b/tests/integration/cases/urllib3_expectation_file/test.yaml new file mode 100644 index 000000000..43bf1fcab --- /dev/null +++ b/tests/integration/cases/urllib3_expectation_file/test.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the repo path when automatic dependency resolution is skipped. + The CUE expectation file is provided as a single file path. + +tags: +- docker + +steps: +- name: Run macaron analyze with expectation file + kind: analyze + options: + expectation: expectation.cue + command_args: + - --repo-path + - https://github.com/urllib3/urllib3/urllib3 + - --branch + - main + - --digest + - 87a0ecee6e691fe5ff93cd000c0158deebef763b + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From fb0a72bd5988a1cd1bfe130b354767461502ff0a Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 20:17:49 +1000 Subject: [PATCH 11/47] chore: convert case urllib3_expectation_dir Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 12 +------- .../dev_scripts/integration_tests_docker.sh | 11 -------- .../expectation/expectation.cue | 11 ++++++++ .../cases/urllib3_expectation_dir/policy.dl | 22 +++++++++++++++ .../cases/urllib3_expectation_dir/test.yaml | 28 +++++++++++++++++++ 5 files changed, 62 insertions(+), 22 deletions(-) create mode 100644 tests/integration/cases/urllib3_expectation_dir/expectation/expectation.cue create mode 100644 tests/integration/cases/urllib3_expectation_dir/policy.dl create mode 100644 tests/integration/cases/urllib3_expectation_dir/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 4b8a392a7..de5be94d9 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -129,16 +129,6 @@ run_macaron_clean $ANALYZE -rp https://github.com/jenkinsci/plot-plugin -b maste $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped." -echo "The CUE expectation file should be found via the directory path." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl -EXPECTATION_DIR=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/ -run_macaron_clean $ANALYZE -pe $EXPECTATION_DIR -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "timyarkov/multibuild_test: Analyzing Maven artifact with the repo path, the branch name and the commit digest" echo "with dependency resolution using cyclonedx Maven plugins (defaults)." @@ -146,8 +136,8 @@ echo -e "----------------------------------------------------------------------- DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_maven.json DEP_RESULT=$WORKSPACE/output/reports/maven/org_example/mock_maven_proj/dependencies.json OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/maven/org.example/mock_maven_proj/1.0-SNAPSHOT/multibuild_test.dl -run_macaron_clean $ANALYZE -purl pkg:maven/org.example/mock_maven_proj@1.0-SNAPSHOT?type=jar -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail +run_macaron_clean $ANALYZE -purl pkg:maven/org.example/mock_maven_proj@1.0-SNAPSHOT?type=jar -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 701d16aca..624132a6f 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -137,17 +137,6 @@ python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail # Clean up and remove the virtual environment. rm -rf "$VIRTUAL_ENV_PATH" -echo -e "\n----------------------------------------------------------------------------------" -echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped." -echo "The CUE expectation file should be found via the directory path." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl -EXPECTATION_DIR=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/ - -run_macaron_clean analyze -pe $EXPECTATION_DIR -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "Test verifying CUE provenance expectation for ossf/scorecard" echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/integration/cases/urllib3_expectation_dir/expectation/expectation.cue b/tests/integration/cases/urllib3_expectation_dir/expectation/expectation.cue new file mode 100644 index 000000000..8f42a812e --- /dev/null +++ b/tests/integration/cases/urllib3_expectation_dir/expectation/expectation.cue @@ -0,0 +1,11 @@ +{ + target: "pkg:github.com/urllib3/urllib3", + predicate: { + invocation: { + configSource: { + uri: =~"^git\\+https://github.com/urllib3/urllib3@refs/tags/v?[0-9]+.[0-9]+.[0-9a-z]+$" + entryPoint: ".github/workflows/publish.yml" + } + } + } +} diff --git a/tests/integration/cases/urllib3_expectation_dir/policy.dl b/tests/integration/cases/urllib3_expectation_dir/policy.dl new file mode 100644 index 000000000..141b722fa --- /dev/null +++ b/tests/integration/cases/urllib3_expectation_dir/policy.dl @@ -0,0 +1,22 @@ +/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_build_as_code_1"), + check_passed(component_id, "mcn_build_script_1"), + check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_provenance_available_1"), + check_passed(component_id, "mcn_provenance_expectation_1"), + check_passed(component_id, "mcn_provenance_level_three_1"), + check_passed(component_id, "mcn_version_control_system_1"), + check_failed(component_id, "mcn_infer_artifact_pipeline_1"), + check_failed(component_id, "mcn_provenance_derived_commit_1"), + check_failed(component_id, "mcn_provenance_derived_repo_1"), + check_failed(component_id, "mcn_provenance_witness_level_one_1"), + check_failed(component_id, "mcn_trusted_builder_level_three_1"), + is_repo_url(component_id, "https://github.com/urllib3/urllib3"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:github.com/urllib3/urllib3@87a0ecee6e691fe5ff93cd000c0158deebef763b"). diff --git a/tests/integration/cases/urllib3_expectation_dir/test.yaml b/tests/integration/cases/urllib3_expectation_dir/test.yaml new file mode 100644 index 000000000..170a0cbcb --- /dev/null +++ b/tests/integration/cases/urllib3_expectation_dir/test.yaml @@ -0,0 +1,28 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the repo path when automatic dependency resolution is skipped. + The CUE expectation file should be found via the directory path. + +tags: +- docker + +steps: +- name: Run macaron analyze with expectation directory + kind: analyze + options: + command_args: + - --repo-path + - https://github.com/urllib3/urllib3/urllib3 + - --branch + - main + - --digest + - 87a0ecee6e691fe5ff93cd000c0158deebef763b + - --provenance-expectation + - expectation + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From d0df6bdb794669d96d409d197f21c6e44de50c24 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 21:59:43 +1000 Subject: [PATCH 12/47] chore: convert case invalid_purl Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 11 ---------- .../dev_scripts/integration_tests_docker.sh | 11 ---------- .../integration/cases/invalid_purl/test.yaml | 20 +++++++++++++++++++ 3 files changed, 20 insertions(+), 22 deletions(-) create mode 100644 tests/integration/cases/invalid_purl/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index de5be94d9..d4d01b55c 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -581,17 +581,6 @@ fi GITHUB_TOKEN="$temp" -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: test analyzing with invalid PURL" -echo -e "----------------------------------------------------------------------------------\n" -run_macaron_clean $ANALYZE -purl invalid-purl -rp https://github.com/apache/maven --skip-deps - -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: test analyzing with both PURL and repository path but no branch and digest are provided." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 624132a6f..066f3119d 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -218,17 +218,6 @@ then fi GITHUB_TOKEN="$temp" -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: test analyzing with invalid PURL" -echo -e "----------------------------------------------------------------------------------\n" -$RUN_MACARON_SCRIPT analyze -purl invalid-purl -rp https://github.com/apache/maven --skip-deps - -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: test analyzing with both PURL and repository path but no branch and digest are provided." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/integration/cases/invalid_purl/test.yaml b/tests/integration/cases/invalid_purl/test.yaml new file mode 100644 index 000000000..8c93f8d52 --- /dev/null +++ b/tests/integration/cases/invalid_purl/test.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: > + Test analyzing with invalid PURL + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - --package-url + - invalid-purl + - --repo-path + - https://github.com/apache/maven + - --skip-deps + expect_fail: true From d0d8f9338dc638927be883d201005e91e45c2063 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 22:15:31 +1000 Subject: [PATCH 13/47] chore: convert case no_github_token Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 14 ------------- .../dev_scripts/integration_tests_docker.sh | 12 ----------- .../cases/no_github_token/test.yaml | 20 +++++++++++++++++++ 3 files changed, 20 insertions(+), 26 deletions(-) create mode 100644 tests/integration/cases/no_github_token/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index d4d01b55c..16106c139 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -567,20 +567,6 @@ then log_fail fi -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: test analyzing without the environment variable GITHUB_TOKEN being set." -echo -e "----------------------------------------------------------------------------------\n" -temp="$GITHUB_TOKEN" -GITHUB_TOKEN="" && run_macaron_clean $ANALYZE -rp https://github.com/apache/maven --skip-deps - -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi - -GITHUB_TOKEN="$temp" - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: test analyzing with both PURL and repository path but no branch and digest are provided." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 066f3119d..57f0954b3 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -206,18 +206,6 @@ $RUN_POLICY -f "$POLICY_FILE" -d $DB || log_fail python "$COMPARE_POLICIES" "$POLICY_RESULT" "$POLICY_EXPECTED" || log_fail python "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "Test running the analysis without setting the GITHUB_TOKEN environment variables." -echo -e "----------------------------------------------------------------------------------\n" -temp="$GITHUB_TOKEN" -GITHUB_TOKEN="" && $RUN_MACARON_SCRIPT analyze -rp https://github.com/slsa-framework/slsa-verifier --skip-deps -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi -GITHUB_TOKEN="$temp" - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: test analyzing with both PURL and repository path but no branch and digest are provided." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/integration/cases/no_github_token/test.yaml b/tests/integration/cases/no_github_token/test.yaml new file mode 100644 index 000000000..3275cfa1a --- /dev/null +++ b/tests/integration/cases/no_github_token/test.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Test running the analysis without setting the GITHUB_TOKEN environment variables. + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - --repo-path + - https://github.com/apache/maven --skip-deps + - --skip-deps + env: + GITHUB_TOKEN: + expect_fail: true From 9a4dcccd7371f0baf1b14fda58745eee8ff2715d Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 22:47:57 +1000 Subject: [PATCH 14/47] chore: convert case ossf_scorecard Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 23 ----------- .../dev_scripts/integration_tests_docker.sh | 24 ------------ .../cases/ossf_scorecard/config.ini} | 0 .../cases/ossf_scorecard/expectation.cue} | 0 .../cases/ossf_scorecard/policy.dl} | 0 .../cases/ossf_scorecard/policy_report.json} | 0 .../cases/ossf_scorecard/test.yaml | 39 +++++++++++++++++++ .../cases/ossf_scorecard}/vsa_payload.json | 0 .../cases/ossf_scorecard/vsa_policy.dl} | 0 9 files changed, 39 insertions(+), 47 deletions(-) rename tests/{e2e/defaults/scorecard.ini => integration/cases/ossf_scorecard/config.ini} (100%) rename tests/{slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue => integration/cases/ossf_scorecard/expectation.cue} (100%) rename tests/{e2e/expected_results/scorecard/scorecard.dl => integration/cases/ossf_scorecard/policy.dl} (100%) rename tests/{policy_engine/expected_results/scorecard/scorecard_policy_report.json => integration/cases/ossf_scorecard/policy_report.json} (100%) create mode 100644 tests/integration/cases/ossf_scorecard/test.yaml rename tests/{vsa/integration/github_slsa-framework_scorecard => integration/cases/ossf_scorecard}/vsa_payload.json (100%) rename tests/{policy_engine/resources/policies/scorecard/scorecard.dl => integration/cases/ossf_scorecard/vsa_policy.dl} (100%) diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 16106c139..e5c2551ce 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -617,29 +617,6 @@ then log_fail fi -# Testing the CUE provenance expectation verifier. -echo -e "\n----------------------------------------------------------------------------------" -echo "Test verifying CUE provenance expectation for ossf/scorecard and run policy CLI" -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.dl -DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/scorecard.ini -EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue -run_macaron_clean -dp $DEFAULTS_FILE $ANALYZE -pe $EXPECTATION_FILE -purl pkg:github/ossf/scorecard@v4.13.1 --skip-deps || log_fail - -# Run CLI policy -POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/scorecard/scorecard.dl -POLICY_RESULT=$WORKSPACE/output/policy_report.json -POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/scorecard/scorecard_policy_report.json -VSA_RESULT=$WORKSPACE/output/vsa.intoto.jsonl -VSA_PAYLOAD_EXPECTED=$WORKSPACE/tests/vsa/integration/github_slsa-framework_scorecard/vsa_payload.json - -$RUN_POLICY -f $POLICY_FILE -d $DB || log_fail -check_or_update_expected_output $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail -check_or_update_expected_output "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail - -# Finish verifying CUE provenance -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "Test verifying CUE provenance expectation for slsa-verifier" echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 57f0954b3..995bf2d63 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -137,30 +137,6 @@ python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail # Clean up and remove the virtual environment. rm -rf "$VIRTUAL_ENV_PATH" -echo -e "\n----------------------------------------------------------------------------------" -echo "Test verifying CUE provenance expectation for ossf/scorecard" -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.dl -DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/scorecard.ini -EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue - -run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/scorecard@v4.13.1 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - -echo -e "\n----------------------------------------------------------------------------------" -echo "Run policy CLI with scorecard results." -echo -e "----------------------------------------------------------------------------------\n" -POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/scorecard/scorecard.dl -POLICY_RESULT=$WORKSPACE/output/policy_report.json -POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/scorecard/scorecard_policy_report.json -VSA_RESULT=$WORKSPACE/output/vsa.intoto.jsonl -VSA_PAYLOAD_EXPECTED=$WORKSPACE/tests/vsa/integration/github_slsa-framework_scorecard/vsa_payload.json - -$RUN_POLICY -f "$POLICY_FILE" -d $DB || log_fail -python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail -python "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped" echo "and CUE file is provided as expectation." diff --git a/tests/e2e/defaults/scorecard.ini b/tests/integration/cases/ossf_scorecard/config.ini similarity index 100% rename from tests/e2e/defaults/scorecard.ini rename to tests/integration/cases/ossf_scorecard/config.ini diff --git a/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue b/tests/integration/cases/ossf_scorecard/expectation.cue similarity index 100% rename from tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue rename to tests/integration/cases/ossf_scorecard/expectation.cue diff --git a/tests/e2e/expected_results/scorecard/scorecard.dl b/tests/integration/cases/ossf_scorecard/policy.dl similarity index 100% rename from tests/e2e/expected_results/scorecard/scorecard.dl rename to tests/integration/cases/ossf_scorecard/policy.dl diff --git a/tests/policy_engine/expected_results/scorecard/scorecard_policy_report.json b/tests/integration/cases/ossf_scorecard/policy_report.json similarity index 100% rename from tests/policy_engine/expected_results/scorecard/scorecard_policy_report.json rename to tests/integration/cases/ossf_scorecard/policy_report.json diff --git a/tests/integration/cases/ossf_scorecard/test.yaml b/tests/integration/cases/ossf_scorecard/test.yaml new file mode 100644 index 000000000..d01a0fef5 --- /dev/null +++ b/tests/integration/cases/ossf_scorecard/test.yaml @@ -0,0 +1,39 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: > + Test CUE provenance expectation for ossf/scorecard, policy verification, and VSA generation. + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + ini: config.ini + expectation: expectation.cue + command_args: + - --package-url + - pkg:github/ossf/scorecard@v4.13.1 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl +- name: Run macaron verify-policy to generate VSA + kind: verify + options: + policy: vsa_policy.dl +- name: Compare verify policy result + kind: compare + options: + kind: policy_report + result: output/policy_report.json + expected: policy_report.json +- name: Compare VSA + kind: compare + options: + kind: vsa + result: output/vsa.intoto.jsonl + expected: vsa_payload.json diff --git a/tests/vsa/integration/github_slsa-framework_scorecard/vsa_payload.json b/tests/integration/cases/ossf_scorecard/vsa_payload.json similarity index 100% rename from tests/vsa/integration/github_slsa-framework_scorecard/vsa_payload.json rename to tests/integration/cases/ossf_scorecard/vsa_payload.json diff --git a/tests/policy_engine/resources/policies/scorecard/scorecard.dl b/tests/integration/cases/ossf_scorecard/vsa_policy.dl similarity index 100% rename from tests/policy_engine/resources/policies/scorecard/scorecard.dl rename to tests/integration/cases/ossf_scorecard/vsa_policy.dl From 84f00c337773e030c56dfb42530db482797e0a5b Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 22:51:43 +1000 Subject: [PATCH 15/47] chore: convert case no_branch_or_commit Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 11 ---------- .../dev_scripts/integration_tests_docker.sh | 11 ---------- .../cases/no_branch_or_commit/test.yaml | 20 +++++++++++++++++++ 3 files changed, 20 insertions(+), 22 deletions(-) create mode 100644 tests/integration/cases/no_branch_or_commit/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index e5c2551ce..2f683519f 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -567,17 +567,6 @@ then log_fail fi -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: test analyzing with both PURL and repository path but no branch and digest are provided." -echo -e "----------------------------------------------------------------------------------\n" -run_macaron_clean $ANALYZE -purl pkg:maven/apache/maven -rp https://github.com/apache/maven --skip-deps - -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi - echo -e "\n-----------------------------------------------------------------------------------------" echo "pkg:pypi/django@5.0.6: Analyzing the dependencies with an invalid path to the virtual env dir." echo -e "-----------------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 995bf2d63..3baa83c91 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -182,17 +182,6 @@ $RUN_POLICY -f "$POLICY_FILE" -d $DB || log_fail python "$COMPARE_POLICIES" "$POLICY_RESULT" "$POLICY_EXPECTED" || log_fail python "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: test analyzing with both PURL and repository path but no branch and digest are provided." -echo -e "----------------------------------------------------------------------------------\n" -$RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven --skip-deps - -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi - python3 ./tests/integration/run.py run \ --macaron scripts/release_scripts/run_macaron.sh \ --include-tag docker \ diff --git a/tests/integration/cases/no_branch_or_commit/test.yaml b/tests/integration/cases/no_branch_or_commit/test.yaml new file mode 100644 index 000000000..8016757bf --- /dev/null +++ b/tests/integration/cases/no_branch_or_commit/test.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: > + Test analyzing with both PURL and repository path but no branch or commit is provided + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - --package-url + - pkg:maven/apache/maven + - --repo-path + - https://github.com/apache/maven + - --skip-deps + expect_fail: true From e82a4809be11bfbeb0b23c8e8990f5e81f888d2a Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 23:16:30 +1000 Subject: [PATCH 16/47] chore: convert case apache_maven_yaml_input_with_dep_resolution Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 9 - .../dev_scripts/integration_tests_docker.sh | 9 - .../dependencies.json | 407 ++++++++++++++++++ .../maven_config.yaml | 9 + .../test.yaml | 22 + 5 files changed, 438 insertions(+), 18 deletions(-) create mode 100644 tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/dependencies.json create mode 100644 tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/maven_config.yaml create mode 100644 tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 2f683519f..df099fc95 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -314,15 +314,6 @@ do $RUN_POLICY -d $DB -f $EXPECT_DIR/$i || log_fail done -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Check the resolved dependency output with config for cyclonedx maven plugin." -echo -e "----------------------------------------------------------------------------------\n" -DEP_RESULT=$WORKSPACE/output/reports/maven/org_apache_maven/maven/dependencies.json -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json -run_macaron_clean $ANALYZE -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || log_fail - -check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Check: Check the e2e status code of running with invalid branch or digest defined in the yaml configuration." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 3baa83c91..a74f177f1 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -61,15 +61,6 @@ $RUN_MACARON_SCRIPT analyze -purl pkg:maven/org.example/mock_gradle_proj@1.0?typ python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Check the resolved dependency output with config for cyclonedx maven plugin (default)." -echo -e "----------------------------------------------------------------------------------\n" -DEP_RESULT=$WORKSPACE/output/reports/maven/org_apache_maven/maven/dependencies.json -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json - -run_macaron_clean analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || log_fail -python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: e2e using the local repo path, the branch name and the commit digest without dependency resolution." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/dependencies.json b/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/dependencies.json new file mode 100644 index 000000000..2b5bf9b7f --- /dev/null +++ b/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/dependencies.json @@ -0,0 +1,407 @@ +[ + { + "id": "org.junit.jupiter:junit-jupiter-api", + "purl": "pkg:maven/org.junit.jupiter/junit-jupiter-api@5.10.0?type=jar", + "path": "https://github.com/junit-team/junit5", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.hamcrest:hamcrest-core", + "purl": "pkg:maven/org.hamcrest/hamcrest-core@2.2?type=jar", + "path": "https://github.com/hamcrest/JavaHamcrest", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.eclipse.sisu:org.eclipse.sisu.plexus", + "purl": "pkg:maven/org.eclipse.sisu/org.eclipse.sisu.plexus@0.9.0.M2?type=jar", + "path": "https://github.com/eclipse/sisu.plexus", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "commons-cli:commons-cli", + "purl": "pkg:maven/commons-cli/commons-cli@1.5.0?type=jar", + "path": "https://github.com/apache/commons-cli", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.wagon:wagon-http", + "purl": "pkg:maven/org.apache.maven.wagon/wagon-http@3.5.3?type=jar", + "path": "https://github.com/apache/maven-wagon", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.wagon:wagon-file", + "purl": "pkg:maven/org.apache.maven.wagon/wagon-file@3.5.3?type=jar", + "path": "https://github.com/apache/maven-wagon", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.slf4j:jcl-over-slf4j", + "purl": "pkg:maven/org.slf4j/jcl-over-slf4j@1.7.36?type=jar", + "path": "https://github.com/qos-ch/slf4j", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.resolver:maven-resolver-connector-basic", + "purl": "pkg:maven/org.apache.maven.resolver/maven-resolver-connector-basic@1.9.16?type=jar", + "path": "https://github.com/apache/maven-resolver", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.resolver:maven-resolver-transport-file", + "purl": "pkg:maven/org.apache.maven.resolver/maven-resolver-transport-file@1.9.16?type=jar", + "path": "https://github.com/apache/maven-resolver", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.resolver:maven-resolver-transport-http", + "purl": "pkg:maven/org.apache.maven.resolver/maven-resolver-transport-http@1.9.16?type=jar", + "path": "https://github.com/apache/maven-resolver", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.resolver:maven-resolver-transport-wagon", + "purl": "pkg:maven/org.apache.maven.resolver/maven-resolver-transport-wagon@1.9.16?type=jar", + "path": "https://github.com/apache/maven-resolver", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.fusesource.jansi:jansi", + "purl": "pkg:maven/org.fusesource.jansi/jansi@2.4.1?type=jar", + "path": "https://github.com/fusesource/jansi", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.ow2.asm:asm", + "purl": "pkg:maven/org.ow2.asm/asm@9.5?type=jar", + "path": "", + "branch": "", + "digest": "", + "note": "Manual configuration required. Could not find SCM URL.", + "available": "MISSING REPO URL" + }, + { + "id": "org.apache.maven.resolver:maven-resolver-api", + "purl": "pkg:maven/org.apache.maven.resolver/maven-resolver-api@1.9.16?type=jar", + "path": "https://github.com/apache/maven-resolver", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.resolver:maven-resolver-util", + "purl": "pkg:maven/org.apache.maven.resolver/maven-resolver-util@1.9.16?type=jar", + "path": "https://github.com/apache/maven-resolver", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.resolver:maven-resolver-impl", + "purl": "pkg:maven/org.apache.maven.resolver/maven-resolver-impl@1.9.16?type=jar", + "path": "https://github.com/apache/maven-resolver", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "javax.inject:javax.inject", + "purl": "pkg:maven/javax.inject/javax.inject@1?type=jar", + "path": "", + "branch": "", + "digest": "", + "note": "Manual configuration required. Could not find SCM URL.", + "available": "MISSING REPO URL" + }, + { + "id": "org.codehaus.plexus:plexus-interpolation", + "purl": "pkg:maven/org.codehaus.plexus/plexus-interpolation@1.26?type=jar", + "path": "https://github.com/codehaus-plexus/plexus-interpolation", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.wagon:wagon-provider-api", + "purl": "pkg:maven/org.apache.maven.wagon/wagon-provider-api@3.5.3?type=jar", + "path": "https://github.com/apache/maven-wagon", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.codehaus.plexus:plexus-testing", + "purl": "pkg:maven/org.codehaus.plexus/plexus-testing@1.0.0?type=jar", + "path": "https://github.com/codehaus-plexus/plexus-testing", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.mockito:mockito-core", + "purl": "pkg:maven/org.mockito/mockito-core@5.2.0?type=jar", + "path": "https://github.com/mockito/mockito", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.apache.maven.resolver:maven-resolver-spi", + "purl": "pkg:maven/org.apache.maven.resolver/maven-resolver-spi@1.9.16?type=jar", + "path": "https://github.com/apache/maven-resolver", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "commons-io:commons-io", + "purl": "pkg:maven/commons-io/commons-io@2.11.0?type=jar", + "path": "https://github.com/apache/commons-io", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.eclipse.sisu:org.eclipse.sisu.inject", + "purl": "pkg:maven/org.eclipse.sisu/org.eclipse.sisu.inject@0.9.0.M2?classifier=no_asm&type=jar", + "path": "https://github.com/eclipse/sisu.inject", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "com.google.inject:guice", + "purl": "pkg:maven/com.google.inject/guice@5.1.0?classifier=classes&type=jar", + "path": "https://github.com/google/guice", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "com.google.guava:guava", + "purl": "pkg:maven/com.google.guava/guava@32.0.1-jre?type=jar", + "path": "https://github.com/google/guava", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "com.google.guava:failureaccess", + "purl": "pkg:maven/com.google.guava/failureaccess@1.0.1?type=jar", + "path": "https://github.com/google/guava", + "branch": "", + "digest": "", + "note": "https://github.com/google/guava is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "org.codehaus.plexus:plexus-classworlds", + "purl": "pkg:maven/org.codehaus.plexus/plexus-classworlds@2.6.0?type=jar", + "path": "https://github.com/codehaus-plexus/plexus-classworlds", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.slf4j:slf4j-api", + "purl": "pkg:maven/org.slf4j/slf4j-api@1.7.36?type=jar", + "path": "https://github.com/qos-ch/slf4j", + "branch": "", + "digest": "", + "note": "https://github.com/qos-ch/slf4j is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "org.slf4j:slf4j-simple", + "purl": "pkg:maven/org.slf4j/slf4j-simple@1.7.36?type=jar", + "path": "https://github.com/qos-ch/slf4j", + "branch": "", + "digest": "", + "note": "https://github.com/qos-ch/slf4j is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "commons-jxpath:commons-jxpath", + "purl": "pkg:maven/commons-jxpath/commons-jxpath@1.3?type=jar", + "path": "", + "branch": "", + "digest": "", + "note": "Manual configuration required. Could not find SCM URL.", + "available": "MISSING REPO URL" + }, + { + "id": "org.mockito:mockito-inline", + "purl": "pkg:maven/org.mockito/mockito-inline@5.2.0?type=jar", + "path": "https://github.com/mockito/mockito", + "branch": "", + "digest": "", + "note": "https://github.com/mockito/mockito is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "org.hamcrest:hamcrest-library", + "purl": "pkg:maven/org.hamcrest/hamcrest-library@2.2?type=jar", + "path": "https://github.com/hamcrest/JavaHamcrest", + "branch": "", + "digest": "", + "note": "https://github.com/hamcrest/JavaHamcrest is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "org.xmlunit:xmlunit-assertj", + "purl": "pkg:maven/org.xmlunit/xmlunit-assertj@2.6.4?type=jar", + "path": "https://github.com/xmlunit/xmlunit", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.junit.jupiter:junit-jupiter-params", + "purl": "pkg:maven/org.junit.jupiter/junit-jupiter-params@5.10.0?type=jar", + "path": "https://github.com/junit-team/junit5", + "branch": "", + "digest": "", + "note": "https://github.com/junit-team/junit5 is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "javax.annotation:javax.annotation-api", + "purl": "pkg:maven/javax.annotation/javax.annotation-api@1.3.2?type=jar", + "path": "https://github.com/javaee/javax.annotation", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.codehaus.plexus:plexus-sec-dispatcher", + "purl": "pkg:maven/org.codehaus.plexus/plexus-sec-dispatcher@2.0?type=jar", + "path": "https://github.com/codehaus-plexus/plexus-sec-dispatcher", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.codehaus.plexus:plexus-cipher", + "purl": "pkg:maven/org.codehaus.plexus/plexus-cipher@2.0?type=jar", + "path": "https://github.com/codehaus-plexus/plexus-cipher", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "ch.qos.logback:logback-classic", + "purl": "pkg:maven/ch.qos.logback/logback-classic@1.2.11?type=jar", + "path": "https://github.com/ceki/logback", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.xmlunit:xmlunit-core", + "purl": "pkg:maven/org.xmlunit/xmlunit-core@2.6.4?type=jar", + "path": "https://github.com/xmlunit/xmlunit", + "branch": "", + "digest": "", + "note": "https://github.com/xmlunit/xmlunit is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "org.xmlunit:xmlunit-matchers", + "purl": "pkg:maven/org.xmlunit/xmlunit-matchers@2.6.4?type=jar", + "path": "https://github.com/xmlunit/xmlunit", + "branch": "", + "digest": "", + "note": "https://github.com/xmlunit/xmlunit is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "org.codehaus.plexus:plexus-xml", + "purl": "pkg:maven/org.codehaus.plexus/plexus-xml@4.0.1?type=jar", + "path": "https://github.com/codehaus-plexus/plexus-xml", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.openjdk.jmh:jmh-core", + "purl": "pkg:maven/org.openjdk.jmh/jmh-core@1.36?type=jar", + "path": "https://github.com/openjdk/jmh", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + }, + { + "id": "org.openjdk.jmh:jmh-generator-annprocess", + "purl": "pkg:maven/org.openjdk.jmh/jmh-generator-annprocess@1.36?type=jar", + "path": "https://github.com/openjdk/jmh", + "branch": "", + "digest": "", + "note": "https://github.com/openjdk/jmh is already analyzed.", + "available": "DUPLICATED REPO URL" + }, + { + "id": "com.fasterxml.woodstox:woodstox-core", + "purl": "pkg:maven/com.fasterxml.woodstox/woodstox-core@6.5.1?type=jar", + "path": "https://github.com/FasterXML/woodstox", + "branch": "", + "digest": "", + "note": "", + "available": "AVAILABLE" + } +] diff --git a/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/maven_config.yaml b/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/maven_config.yaml new file mode 100644 index 000000000..091665bc0 --- /dev/null +++ b/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/maven_config.yaml @@ -0,0 +1,9 @@ +# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +target: + id: apache/maven + branch: master + digest: 3fc399318edef0d5ba593723a24fff64291d6f9b + path: https://github.com/apache/maven.git + purl: pkg:maven/org.apache.maven/maven@4.0.0-alpha-9-SNAPSHOT?type=pom diff --git a/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/test.yaml b/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/test.yaml new file mode 100644 index 000000000..81d1cfc39 --- /dev/null +++ b/tests/integration/cases/apache_maven_yaml_input_with_dep_resolution/test.yaml @@ -0,0 +1,22 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Check the resolved dependency output with config for cyclonedx maven plugin + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -c + - maven_config.yaml +- name: Compare dependency report + kind: compare + options: + kind: deps_report + result: output/reports/maven/org_apache_maven/maven/dependencies.json + expected: dependencies.json From d5cc7e0a4e21dcded20980c7526f3bcc7a357c20 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 23:22:36 +1000 Subject: [PATCH 17/47] chore: convert case apache_maven_local_repo Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 10 ------ .../dev_scripts/integration_tests_docker.sh | 9 ----- .../cases/apache_maven_local_repo/policy.dl | 22 ++++++++++++ .../cases/apache_maven_local_repo/test.yaml | 36 +++++++++++++++++++ 4 files changed, 58 insertions(+), 19 deletions(-) create mode 100644 tests/integration/cases/apache_maven_local_repo/policy.dl create mode 100644 tests/integration/cases/apache_maven_local_repo/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index df099fc95..6ca709da8 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -424,16 +424,6 @@ do $RUN_POLICY -d $DB -f $EXPECT_DIR/$i || log_fail done -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Analyzing with local paths using local_repos_dir without dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -# The report files are still stored in the same location. -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/maven/maven.dl - -run_macaron_clean -lr $WORKSPACE/output/git_repos/github_com/ $ANALYZE -rp apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Analyzing a repository that was cloned from another local repo." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index a74f177f1..0a9b1897d 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -61,15 +61,6 @@ $RUN_MACARON_SCRIPT analyze -purl pkg:maven/org.example/mock_gradle_proj@1.0?typ python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: e2e using the local repo path, the branch name and the commit digest without dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/maven/maven.dl - -run_macaron_clean -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Check the e2e output JSON file with config and no dependency analyzing." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/integration/cases/apache_maven_local_repo/policy.dl b/tests/integration/cases/apache_maven_local_repo/policy.dl new file mode 100644 index 000000000..ef16459c9 --- /dev/null +++ b/tests/integration/cases/apache_maven_local_repo/policy.dl @@ -0,0 +1,22 @@ +/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_build_script_1"), + check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_version_control_system_1"), + check_failed(component_id, "mcn_build_as_code_1"), + check_failed(component_id, "mcn_infer_artifact_pipeline_1"), + check_failed(component_id, "mcn_provenance_available_1"), + check_failed(component_id, "mcn_provenance_derived_commit_1"), + check_failed(component_id, "mcn_provenance_derived_repo_1"), + check_failed(component_id, "mcn_provenance_expectation_1"), + check_failed(component_id, "mcn_provenance_level_three_1"), + check_failed(component_id, "mcn_provenance_witness_level_one_1"), + check_failed(component_id, "mcn_trusted_builder_level_three_1"), + is_repo_url(component_id, "https://github.com/apache/maven"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:github.com/apache/maven@3fc399318edef0d5ba593723a24fff64291d6f9b"). diff --git a/tests/integration/cases/apache_maven_local_repo/test.yaml b/tests/integration/cases/apache_maven_local_repo/test.yaml new file mode 100644 index 000000000..281a1997e --- /dev/null +++ b/tests/integration/cases/apache_maven_local_repo/test.yaml @@ -0,0 +1,36 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing with local paths using local_repos_dir without dependency resolution. + +tags: +- docker + +steps: +- name: Prepare local repo directory + kind: shell + options: + cmd: mkdir -p output/git_repos/github_com/apache +- name: Clone apache/maven + kind: shell + options: + cmd: git clone --filter=tree:0 https://github.com/apache/maven output/git_repos/github_com/apache/maven +- name: Run macaron analyze + kind: analyze + options: + main_args: + - -lr + - output/git_repos/github_com + command_args: + - -r + - apache/maven + - -b + - master + - -d + - 3fc399318edef0d5ba593723a24fff64291d6f9b + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 66a4dfc8ad8b53466a11ee4fc3f8e11d80e3faca Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 15 Jun 2024 23:47:48 +1000 Subject: [PATCH 18/47] chore: convert case timyarkov_multibuild_test_gradle Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 10 ------- .../dev_scripts/integration_tests_docker.sh | 10 ------- .../dependencies.json} | 0 .../test.yaml | 28 +++++++++++++++++++ 4 files changed, 28 insertions(+), 20 deletions(-) rename tests/{dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_gradle.json => integration/cases/timyarkov_multibuild_test_gradle/dependencies.json} (100%) create mode 100644 tests/integration/cases/timyarkov_multibuild_test_gradle/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 6ca709da8..f280726ec 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -142,16 +142,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "timyarkov/multibuild_test: Analyzing Gradle artifact with the repo path, the branch name and the commit digest" -echo "with dependency resolution using cyclonedx Gradle plugins (defaults)." -echo -e "----------------------------------------------------------------------------------\n" -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_gradle.json -DEP_RESULT=$WORKSPACE/output/reports/maven/org_example/mock_gradle_proj/dependencies.json -run_macaron_clean $ANALYZE -purl pkg:maven/org.example/mock_gradle_proj@1.0?type=jar -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail - -check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - echo "timyarkov/docker_test: Analyzing the repo path, the branch name and the commit digest" echo "when automatic dependency resolution is skipped, for a project using docker as a build tool." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 0a9b1897d..4a9ef2240 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -51,16 +51,6 @@ python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "timyarkov/multibuild_test: Analyzing Gradle artifact with the repo path, the branch name and the commit digest" -echo "with dependency resolution using cyclonedx Gradle plugins (defaults)." -echo -e "----------------------------------------------------------------------------------\n" -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_gradle.json -DEP_RESULT=$WORKSPACE/output/reports/maven/org_example/mock_gradle_proj/dependencies.json -$RUN_MACARON_SCRIPT analyze -purl pkg:maven/org.example/mock_gradle_proj@1.0?type=jar -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail - -python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Check the e2e output JSON file with config and no dependency analyzing." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_gradle.json b/tests/integration/cases/timyarkov_multibuild_test_gradle/dependencies.json similarity index 100% rename from tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_gradle.json rename to tests/integration/cases/timyarkov_multibuild_test_gradle/dependencies.json diff --git a/tests/integration/cases/timyarkov_multibuild_test_gradle/test.yaml b/tests/integration/cases/timyarkov_multibuild_test_gradle/test.yaml new file mode 100644 index 000000000..69a0303b2 --- /dev/null +++ b/tests/integration/cases/timyarkov_multibuild_test_gradle/test.yaml @@ -0,0 +1,28 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyze with dependency resolution using cyclonedx Gradle plugin (default) + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/org.example/mock_gradle_proj@1.0?type=jar + - -rp + - https://github.com/timyarkov/multibuild_test + - -b + - main + - -d + - a8b0efe24298bc81f63217aaa84776c3d48976c5 +- name: Compare dependency resolution result + kind: compare + options: + kind: deps_report + result: output/reports/maven/org_example/mock_gradle_proj/dependencies.json + expected: dependencies.json From fa679a887fb3f011982cc59cc64bcc8f34610023 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 00:05:34 +1000 Subject: [PATCH 19/47] chore: convert case apache_maven_purl_repo_path Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 8 ------ .../dev_scripts/integration_tests_docker.sh | 9 ------- .../apache_maven_purl_repo_path/policy.dl} | 0 .../apache_maven_purl_repo_path/test.yaml | 27 +++++++++++++++++++ 4 files changed, 27 insertions(+), 17 deletions(-) rename tests/{e2e/expected_results/purl/maven/maven.dl => integration/cases/apache_maven_purl_repo_path/policy.dl} (100%) create mode 100644 tests/integration/cases/apache_maven_purl_repo_path/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index f280726ec..8be0ac596 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -195,14 +195,6 @@ run_macaron_clean $ANALYZE -rp https://github.com/sigstore/sget -b main -d 99e7b $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Analyzing with PURL and repository path without dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.dl -run_macaron_clean $ANALYZE -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Analyzing the repo path, the branch name and the commit digest with dependency resolution using cyclonedx maven plugin (default)." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 4a9ef2240..1b2a65d7c 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -80,15 +80,6 @@ run_macaron_clean analyze -purl pkg:maven/org.apache.maven/maven@4.0.0-alpha-1-S python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Analyzing with PURL and repository path without dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.dl - -run_macaron_clean analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "pkg:pypi/django@5.0.6: Analyzing the dependencies with virtual env provided as input." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/e2e/expected_results/purl/maven/maven.dl b/tests/integration/cases/apache_maven_purl_repo_path/policy.dl similarity index 100% rename from tests/e2e/expected_results/purl/maven/maven.dl rename to tests/integration/cases/apache_maven_purl_repo_path/policy.dl diff --git a/tests/integration/cases/apache_maven_purl_repo_path/test.yaml b/tests/integration/cases/apache_maven_purl_repo_path/test.yaml new file mode 100644 index 000000000..55773b236 --- /dev/null +++ b/tests/integration/cases/apache_maven_purl_repo_path/test.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyze with PURL, repository path, no dependency resolution. + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - --package-url + - pkg:maven/apache/maven + - --repo-path + - https://github.com/apache/maven + - -b + - master + - -d + - 3fc399318edef0d5ba593723a24fff64291d6f9b + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 9ea116cccd0e1a649948943026b8e2128ec43e6f Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 00:20:57 +1000 Subject: [PATCH 20/47] chore: convert case behnazh-w_example-maven-app Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 42 ---------- .../dev_scripts/integration_tests_docker.sh | 33 -------- .../github_provenance.jsonl} | 0 .../github_provenance_expectation.cue} | 0 .../behnazh-w_example-maven-app}/policy.dl | 0 .../policy_report.json} | 0 .../behnazh-w_example-maven-app/test.yaml | 79 +++++++++++++++++++ .../vsa_payload.json | 0 .../witness_provenance.jsonl} | 0 .../witness_provenance_expectation.cue} | 0 10 files changed, 79 insertions(+), 75 deletions(-) rename tests/{slsa_analyzer/provenance/resources/valid_provenances/github-example-maven-project.json => integration/cases/behnazh-w_example-maven-app/github_provenance.jsonl} (100%) rename tests/{slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/github-example-maven-project.cue => integration/cases/behnazh-w_example-maven-app/github_provenance_expectation.cue} (100%) rename tests/{policy_engine/resources/policies/example-maven-project => integration/cases/behnazh-w_example-maven-app}/policy.dl (100%) rename tests/{policy_engine/expected_results/example-maven-project/example_maven_project_policy_report.json => integration/cases/behnazh-w_example-maven-app/policy_report.json} (100%) create mode 100644 tests/integration/cases/behnazh-w_example-maven-app/test.yaml rename tests/{vsa/integration/example-maven-project => integration/cases/behnazh-w_example-maven-app}/vsa_payload.json (100%) rename tests/{slsa_analyzer/provenance/resources/valid_provenances/witness-example-maven-project.json => integration/cases/behnazh-w_example-maven-app/witness_provenance.jsonl} (100%) rename tests/{slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/witness-example-maven-project.cue => integration/cases/behnazh-w_example-maven-app/witness_provenance_expectation.cue} (100%) diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 8be0ac596..30b1ca6ee 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -623,48 +623,6 @@ $RUN_MACARON -dp $DEFAULTS_FILE analyze -purl pkg:maven/io.micronaut/micronaut-c $RUN_POLICY -f $POLICY_FILE -d $DB || log_fail check_or_update_expected_output $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "behnazh-w/example-maven-app as a local and remote repository" -echo "Test the Witness and GitHub provenances as an input, Cue expectation validation, Policy CLI and VSA generation, User input vs. provenance." -echo -e "----------------------------------------------------------------------------------\n" -POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/example-maven-project/policy.dl -POLICY_RESULT=$WORKSPACE/output/policy_report.json -POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/example-maven-project/example_maven_project_policy_report.json -VSA_RESULT=$WORKSPACE/output/vsa.intoto.jsonl -VSA_PAYLOAD_EXPECTED=$WORKSPACE/tests/vsa/integration/example-maven-project/vsa_payload.json - -# Test the local repo with Witness provenance. -WITNESS_EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/witness-example-maven-project.cue -WITNESS_PROVENANCE_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/resources/valid_provenances/witness-example-maven-project.json - -# Cloning the repository locally -git clone https://github.com/behnazh-w/example-maven-app.git $WORKSPACE/output/git_repos/local_repos/example-maven-app || log_fail - -# Check the Witness provenance. -run_macaron_clean $ANALYZE -pf $WITNESS_PROVENANCE_FILE -pe $WITNESS_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0-SNAPSHOT?type=jar --repo-path example-maven-app --skip-deps || log_fail - -# Test the remote repo with GitHub provenance. -GITHUB_EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/github-example-maven-project.cue -GITHUB_PROVENANCE_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/resources/valid_provenances/github-example-maven-project.json - -# Check the GitHub provenance (Do not clean). -$RUN_MACARON $ANALYZE -pf $GITHUB_PROVENANCE_FILE -pe $GITHUB_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0?type=jar --skip-deps || log_fail - -# Verify the policy and VSA for all the software components generated from behnazh-w/example-maven-app repo. -$RUN_POLICY -f $POLICY_FILE -d $DB || log_fail - -check_or_update_expected_output "$COMPARE_POLICIES" "$POLICY_RESULT" "$POLICY_EXPECTED" || log_fail -check_or_update_expected_output "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail - -# Validate user input of repo and commit vs provenance. -run_macaron_clean $ANALYZE -pf $GITHUB_PROVENANCE_FILE -rp https://github.com/behnazh-w/example-maven-app -d 2deca75ed5dd365eaf1558a82347b1f11306135f --skip-deps || log_fail - -# Validate user input of repo and commit (via purl) vs provenance. -run_macaron_clean $ANALYZE -pf $GITHUB_PROVENANCE_FILE -purl pkg:github/behnazh-w/example-maven-app@2deca75 --skip-deps || log_fail - -# Validate user input of repo and commit (via purl with tag) vs provenance. -run_macaron_clean $ANALYZE -pf $GITHUB_PROVENANCE_FILE -purl pkg:github/behnazh-w/example-maven-app@1.0 --skip-deps || log_fail - # Testing the Repo Finder's remote calls. # This requires the 'packageurl' Python module echo -e "\n----------------------------------------------------------------------------------" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 1b2a65d7c..3be739f18 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -112,39 +112,6 @@ run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://g $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "behnazh-w/example-maven-app as a local and remote repository" -echo "Test the Witness and GitHub provenances as an input, Cue expectation validation, Policy CLI and VSA generation." -echo -e "----------------------------------------------------------------------------------\n" -POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/example-maven-project/policy.dl -POLICY_RESULT=$WORKSPACE/output/policy_report.json -POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/example-maven-project/example_maven_project_policy_report.json -VSA_RESULT=$WORKSPACE/output/vsa.intoto.jsonl -VSA_PAYLOAD_EXPECTED=$WORKSPACE/tests/vsa/integration/example-maven-project/vsa_payload.json - -# Test the local repo with Witness provenance. -WITNESS_EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/witness-example-maven-project.cue -WITNESS_PROVENANCE_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/resources/valid_provenances/witness-example-maven-project.json - -# Cloning the repository locally -git clone https://github.com/behnazh-w/example-maven-app.git $WORKSPACE/output/git_repos/local_repos/example-maven-app || log_fail - -# Check the Witness provenance. -run_macaron_clean analyze -pf $WITNESS_PROVENANCE_FILE -pe $WITNESS_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0-SNAPSHOT?type=jar --repo-path example-maven-app --skip-deps || log_fail - -# Test the remote repo with GitHub provenance. -GITHUB_EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/github-example-maven-project.cue -GITHUB_PROVENANCE_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/resources/valid_provenances/github-example-maven-project.json - -# Check the GitHub provenance. -$RUN_MACARON_SCRIPT analyze -pf $GITHUB_PROVENANCE_FILE -pe $GITHUB_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0?type=jar --skip-deps || log_fail - -# Verify the policy and VSA for all the software components generated from behnazh-w/example-maven-app repo. -$RUN_POLICY -f "$POLICY_FILE" -d $DB || log_fail - -python "$COMPARE_POLICIES" "$POLICY_RESULT" "$POLICY_EXPECTED" || log_fail -python "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail - python3 ./tests/integration/run.py run \ --macaron scripts/release_scripts/run_macaron.sh \ --include-tag docker \ diff --git a/tests/slsa_analyzer/provenance/resources/valid_provenances/github-example-maven-project.json b/tests/integration/cases/behnazh-w_example-maven-app/github_provenance.jsonl similarity index 100% rename from tests/slsa_analyzer/provenance/resources/valid_provenances/github-example-maven-project.json rename to tests/integration/cases/behnazh-w_example-maven-app/github_provenance.jsonl diff --git a/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/github-example-maven-project.cue b/tests/integration/cases/behnazh-w_example-maven-app/github_provenance_expectation.cue similarity index 100% rename from tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/github-example-maven-project.cue rename to tests/integration/cases/behnazh-w_example-maven-app/github_provenance_expectation.cue diff --git a/tests/policy_engine/resources/policies/example-maven-project/policy.dl b/tests/integration/cases/behnazh-w_example-maven-app/policy.dl similarity index 100% rename from tests/policy_engine/resources/policies/example-maven-project/policy.dl rename to tests/integration/cases/behnazh-w_example-maven-app/policy.dl diff --git a/tests/policy_engine/expected_results/example-maven-project/example_maven_project_policy_report.json b/tests/integration/cases/behnazh-w_example-maven-app/policy_report.json similarity index 100% rename from tests/policy_engine/expected_results/example-maven-project/example_maven_project_policy_report.json rename to tests/integration/cases/behnazh-w_example-maven-app/policy_report.json diff --git a/tests/integration/cases/behnazh-w_example-maven-app/test.yaml b/tests/integration/cases/behnazh-w_example-maven-app/test.yaml new file mode 100644 index 000000000..6baa4f7bc --- /dev/null +++ b/tests/integration/cases/behnazh-w_example-maven-app/test.yaml @@ -0,0 +1,79 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Test the Witness and GitHub provenances as an input, Cue expectation validation, + Policy CLI, and VSA generation. + +tags: +- docker + +steps: +- name: Clone the repository + kind: shell + options: + cmd: > + git clone https://github.com/behnazh-w/example-maven-app.git + output/git_repos/local_repos/example-maven-app +- name: Run macaron analyze on the local repository with Witness provenance. + kind: analyze + options: + command_args: + - --package-url + - pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0-SNAPSHOT?type=jar + - --repo-path + - example-maven-app + - --skip-deps + expectation: witness_provenance_expectation.cue + provenance: witness_provenance.jsonl +- name: Run macaron analyze on the remote repository with GitHub provenance. + kind: analyze + options: + command_args: + - --package-url + - pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0?type=jar + - --skip-deps + expectation: github_provenance_expectation.cue + provenance: github_provenance.jsonl +- name: Run macaron verify-policy + kind: verify + options: + policy: policy.dl +- name: Compare verify report + kind: compare + options: + kind: policy_report + result: output/policy_report.json + expected: policy_report.json +- name: Compare VSA + kind: compare + options: + kind: vsa + result: output/vsa.intoto.jsonl + expected: vsa_payload.json +- name: Run macaron analyze, validate user input of repo and commit vs provenance + kind: analyze + options: + command_args: + - -rp + - https://github.com/behnazh-w/example-maven-app + - -d + - 2deca75ed5dd365eaf1558a82347b1f11306135f + - --skip-deps + provenance: github_provenance.jsonl +- name: Run macaron analyze, validate user input of repo and commit (via purl) vs provenance + kind: analyze + options: + command_args: + - -purl + - pkg:github/behnazh-w/example-maven-app@2deca75 + - --skip-deps + provenance: github_provenance.jsonl +- name: Run macaron analyze, validate user input of repo and commit (via purl with tag) vs provenance + kind: analyze + options: + command_args: + - -purl + - pkg:github/behnazh-w/example-maven-app@1.0 + - --skip-deps + provenance: github_provenance.jsonl diff --git a/tests/vsa/integration/example-maven-project/vsa_payload.json b/tests/integration/cases/behnazh-w_example-maven-app/vsa_payload.json similarity index 100% rename from tests/vsa/integration/example-maven-project/vsa_payload.json rename to tests/integration/cases/behnazh-w_example-maven-app/vsa_payload.json diff --git a/tests/slsa_analyzer/provenance/resources/valid_provenances/witness-example-maven-project.json b/tests/integration/cases/behnazh-w_example-maven-app/witness_provenance.jsonl similarity index 100% rename from tests/slsa_analyzer/provenance/resources/valid_provenances/witness-example-maven-project.json rename to tests/integration/cases/behnazh-w_example-maven-app/witness_provenance.jsonl diff --git a/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/witness-example-maven-project.cue b/tests/integration/cases/behnazh-w_example-maven-app/witness_provenance_expectation.cue similarity index 100% rename from tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/witness-example-maven-project.cue rename to tests/integration/cases/behnazh-w_example-maven-app/witness_provenance_expectation.cue From f0d10fe1a68dfe0254cd420058867c7a26d07367 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 00:31:53 +1000 Subject: [PATCH 21/47] chore: convert case apache_maven_yaml_input_skip_deps Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 18 ------------ .../dev_scripts/integration_tests_docker.sh | 18 ------------ .../guava.dl | 22 ++++++++++++++ .../maven.dl | 22 ++++++++++++++ .../maven_config.yaml | 2 +- .../mockito.dl | 22 ++++++++++++++ .../test.yaml | 29 +++++++++++++++++++ 7 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 tests/integration/cases/apache_maven_yaml_input_skip_deps/guava.dl create mode 100644 tests/integration/cases/apache_maven_yaml_input_skip_deps/maven.dl rename tests/{e2e/configurations => integration/cases/apache_maven_yaml_input_skip_deps}/maven_config.yaml (89%) create mode 100644 tests/integration/cases/apache_maven_yaml_input_skip_deps/mockito.dl create mode 100644 tests/integration/cases/apache_maven_yaml_input_skip_deps/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 30b1ca6ee..6a4880192 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -278,24 +278,6 @@ run_macaron_clean $ANALYZE -c $WORKSPACE/tests/dependency_analyzer/configuration check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Check the e2e results with config and no dependency analyzing." -echo -e "----------------------------------------------------------------------------------\n" -EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/maven - -declare -a COMPARE_FILES=( - "maven.dl" - "guava.dl" - "mockito.dl" -) - -run_macaron_clean $ANALYZE -c $WORKSPACE/tests/e2e/configurations/maven_config.yaml --skip-deps || log_fail - -for i in "${COMPARE_FILES[@]}" -do - $RUN_POLICY -d $DB -f $EXPECT_DIR/$i || log_fail -done - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Check: Check the e2e status code of running with invalid branch or digest defined in the yaml configuration." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 3be739f18..c4c7d36b7 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -51,24 +51,6 @@ python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Check the e2e output JSON file with config and no dependency analyzing." -echo -e "----------------------------------------------------------------------------------\n" -EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/maven - -declare -a COMPARE_FILES=( - "maven.dl" - "guava.dl" - "mockito.dl" -) - -run_macaron_clean analyze -c $WORKSPACE/tests/e2e/configurations/maven_config.yaml --skip-deps || log_fail - -for i in "${COMPARE_FILES[@]}" -do - $RUN_POLICY -d $DB -f $EXPECT_DIR/$i || log_fail -done - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Analyzing using a CycloneDx SBOM with target repo path" echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/integration/cases/apache_maven_yaml_input_skip_deps/guava.dl b/tests/integration/cases/apache_maven_yaml_input_skip_deps/guava.dl new file mode 100644 index 000000000..5f5927982 --- /dev/null +++ b/tests/integration/cases/apache_maven_yaml_input_skip_deps/guava.dl @@ -0,0 +1,22 @@ +/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_build_as_code_1"), + check_passed(component_id, "mcn_build_script_1"), + check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_version_control_system_1"), + check_failed(component_id, "mcn_infer_artifact_pipeline_1"), + check_failed(component_id, "mcn_provenance_available_1"), + check_failed(component_id, "mcn_provenance_derived_commit_1"), + check_failed(component_id, "mcn_provenance_derived_repo_1"), + check_failed(component_id, "mcn_provenance_expectation_1"), + check_failed(component_id, "mcn_provenance_level_three_1"), + check_failed(component_id, "mcn_provenance_witness_level_one_1"), + check_failed(component_id, "mcn_trusted_builder_level_three_1"), + is_repo_url(component_id, "https://github.com/google/guava"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:github.com/google/guava@d8633ac8539dae52c8361f79c7a0dbd9ad6dd2c4"). diff --git a/tests/integration/cases/apache_maven_yaml_input_skip_deps/maven.dl b/tests/integration/cases/apache_maven_yaml_input_skip_deps/maven.dl new file mode 100644 index 000000000..ef16459c9 --- /dev/null +++ b/tests/integration/cases/apache_maven_yaml_input_skip_deps/maven.dl @@ -0,0 +1,22 @@ +/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_build_script_1"), + check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_version_control_system_1"), + check_failed(component_id, "mcn_build_as_code_1"), + check_failed(component_id, "mcn_infer_artifact_pipeline_1"), + check_failed(component_id, "mcn_provenance_available_1"), + check_failed(component_id, "mcn_provenance_derived_commit_1"), + check_failed(component_id, "mcn_provenance_derived_repo_1"), + check_failed(component_id, "mcn_provenance_expectation_1"), + check_failed(component_id, "mcn_provenance_level_three_1"), + check_failed(component_id, "mcn_provenance_witness_level_one_1"), + check_failed(component_id, "mcn_trusted_builder_level_three_1"), + is_repo_url(component_id, "https://github.com/apache/maven"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:github.com/apache/maven@3fc399318edef0d5ba593723a24fff64291d6f9b"). diff --git a/tests/e2e/configurations/maven_config.yaml b/tests/integration/cases/apache_maven_yaml_input_skip_deps/maven_config.yaml similarity index 89% rename from tests/e2e/configurations/maven_config.yaml rename to tests/integration/cases/apache_maven_yaml_input_skip_deps/maven_config.yaml index 345cb1cfb..e07caca56 100644 --- a/tests/e2e/configurations/maven_config.yaml +++ b/tests/integration/cases/apache_maven_yaml_input_skip_deps/maven_config.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: diff --git a/tests/integration/cases/apache_maven_yaml_input_skip_deps/mockito.dl b/tests/integration/cases/apache_maven_yaml_input_skip_deps/mockito.dl new file mode 100644 index 000000000..f754eb3e5 --- /dev/null +++ b/tests/integration/cases/apache_maven_yaml_input_skip_deps/mockito.dl @@ -0,0 +1,22 @@ +/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_passed(component_id, "mcn_build_as_code_1"), + check_passed(component_id, "mcn_build_script_1"), + check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_version_control_system_1"), + check_failed(component_id, "mcn_infer_artifact_pipeline_1"), + check_failed(component_id, "mcn_provenance_available_1"), + check_failed(component_id, "mcn_provenance_derived_commit_1"), + check_failed(component_id, "mcn_provenance_derived_repo_1"), + check_failed(component_id, "mcn_provenance_expectation_1"), + check_failed(component_id, "mcn_provenance_level_three_1"), + check_failed(component_id, "mcn_provenance_witness_level_one_1"), + check_failed(component_id, "mcn_trusted_builder_level_three_1"), + is_repo_url(component_id, "https://github.com/mockito/mockito"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:github.com/mockito/mockito@512ee3949484e4765038a0410cd7a7f1b73cc655"). diff --git a/tests/integration/cases/apache_maven_yaml_input_skip_deps/test.yaml b/tests/integration/cases/apache_maven_yaml_input_skip_deps/test.yaml new file mode 100644 index 000000000..b0136065e --- /dev/null +++ b/tests/integration/cases/apache_maven_yaml_input_skip_deps/test.yaml @@ -0,0 +1,29 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Check the e2e output JSON file with config and no dependency analyzing. + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -c + - maven_config.yaml + - --skip-deps +- name: Run macaron verify-policy for maven + kind: verify + options: + policy: maven.dl +- name: Run macaron verify-policy for guava + kind: verify + options: + policy: guava.dl +- name: Run macaron verify-policy for mockito + kind: verify + options: + policy: mockito.dl From 130cda000618c5942f9dee2a43ec7ba0374238be Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 00:47:31 +1000 Subject: [PATCH 22/47] chore: convert case slsa-framework_slsa-verifier Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 10 ------- .../dev_scripts/integration_tests_docker.sh | 12 --------- .../slsa-framework_slsa-verifier/config.ini | 6 +++++ .../expectation.cue | 11 ++++++++ .../slsa-framework_slsa-verifier/policy.dl} | 0 .../slsa-framework_slsa-verifier/test.yaml | 27 +++++++++++++++++++ 6 files changed, 44 insertions(+), 22 deletions(-) create mode 100644 tests/integration/cases/slsa-framework_slsa-verifier/config.ini create mode 100644 tests/integration/cases/slsa-framework_slsa-verifier/expectation.cue rename tests/{e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.dl => integration/cases/slsa-framework_slsa-verifier/policy.dl} (100%) create mode 100644 tests/integration/cases/slsa-framework_slsa-verifier/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 6a4880192..9994e550e 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -551,16 +551,6 @@ then log_fail fi -echo -e "\n----------------------------------------------------------------------------------" -echo "Test verifying CUE provenance expectation for slsa-verifier" -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.dl -EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue -DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini -run_macaron_clean -dp $DEFAULTS_FILE $ANALYZE -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "Test verifying CUE provenance expectation for slsa-verifier with explicitly-provided provenance file" echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index c4c7d36b7..54eee6014 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -82,18 +82,6 @@ python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail # Clean up and remove the virtual environment. rm -rf "$VIRTUAL_ENV_PATH" -echo -e "\n----------------------------------------------------------------------------------" -echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped" -echo "and CUE file is provided as expectation." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.dl -EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue -DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini - -run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - python3 ./tests/integration/run.py run \ --macaron scripts/release_scripts/run_macaron.sh \ --include-tag docker \ diff --git a/tests/integration/cases/slsa-framework_slsa-verifier/config.ini b/tests/integration/cases/slsa-framework_slsa-verifier/config.ini new file mode 100644 index 000000000..884ca6874 --- /dev/null +++ b/tests/integration/cases/slsa-framework_slsa-verifier/config.ini @@ -0,0 +1,6 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +[analysis.checks] +exclude = mcn_provenance_level_three_1 +include = * diff --git a/tests/integration/cases/slsa-framework_slsa-verifier/expectation.cue b/tests/integration/cases/slsa-framework_slsa-verifier/expectation.cue new file mode 100644 index 000000000..24addda8f --- /dev/null +++ b/tests/integration/cases/slsa-framework_slsa-verifier/expectation.cue @@ -0,0 +1,11 @@ +{ + target: "pkg:github.com/slsa-framework/slsa-verifier", + predicate: { + invocation: { + configSource: { + uri: =~"^git\\+https://github.com/slsa-framework/slsa-verifier@refs/tags/v[0-9]+.[0-9]+.[0-9a-z]+$" + entryPoint: ".github/workflows/release.yml" + } + } + } +} diff --git a/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.dl b/tests/integration/cases/slsa-framework_slsa-verifier/policy.dl similarity index 100% rename from tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.dl rename to tests/integration/cases/slsa-framework_slsa-verifier/policy.dl diff --git a/tests/integration/cases/slsa-framework_slsa-verifier/test.yaml b/tests/integration/cases/slsa-framework_slsa-verifier/test.yaml new file mode 100644 index 000000000..54e05c2a8 --- /dev/null +++ b/tests/integration/cases/slsa-framework_slsa-verifier/test.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Test CUE provenance expectation check and policy verification. + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + ini: config.ini + expectation: expectation.cue + command_args: + - -rp + - https://github.com/slsa-framework/slsa-verifier + - -b + - main + - -d + - fc50b662fcfeeeb0e97243554b47d9b20b14efac + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 14d0648e773384f611ba302e32133ed69712ff12 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 00:58:57 +1000 Subject: [PATCH 23/47] chore: convert case apache_maven_sbom Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 10 ------- .../dev_scripts/integration_tests_docker.sh | 11 ------- .../apache_maven_sbom/dependencies.json} | 0 .../cases/apache_maven_sbom/sbom.json} | 0 .../cases/apache_maven_sbom/test.yaml | 29 +++++++++++++++++++ 5 files changed, 29 insertions(+), 21 deletions(-) rename tests/{dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json => integration/cases/apache_maven_sbom/dependencies.json} (100%) rename tests/{dependency_analyzer/cyclonedx/resources/apache_maven_root_sbom.json => integration/cases/apache_maven_sbom/sbom.json} (100%) create mode 100644 tests/integration/cases/apache_maven_sbom/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 9994e550e..f5c001fc4 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -207,16 +207,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Analyzing using a CycloneDx SBOM with target repo path" -echo -e "----------------------------------------------------------------------------------\n" -SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_root_sbom.json -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json -DEP_RESULT=$WORKSPACE/output/reports/maven/org_apache_maven/maven/dependencies.json -run_macaron_clean $ANALYZE -purl pkg:maven/org.apache.maven/maven@4.0.0-alpha-1-SNAPSHOT?type=pom -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom "$SBOM_FILE" || log_fail - -check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "com.example/nonexistent: Analyzing purl of nonexistent artifact." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 54eee6014..9c333b131 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -51,17 +51,6 @@ python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Analyzing using a CycloneDx SBOM with target repo path" -echo -e "----------------------------------------------------------------------------------\n" -SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_root_sbom.json -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json -DEP_RESULT=$WORKSPACE/output/reports/maven/org_apache_maven/maven/dependencies.json - -run_macaron_clean analyze -purl pkg:maven/org.apache.maven/maven@4.0.0-alpha-1-SNAPSHOT?type=pom -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom "$SBOM_FILE" || log_fail - -python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "pkg:pypi/django@5.0.6: Analyzing the dependencies with virtual env provided as input." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json b/tests/integration/cases/apache_maven_sbom/dependencies.json similarity index 100% rename from tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json rename to tests/integration/cases/apache_maven_sbom/dependencies.json diff --git a/tests/dependency_analyzer/cyclonedx/resources/apache_maven_root_sbom.json b/tests/integration/cases/apache_maven_sbom/sbom.json similarity index 100% rename from tests/dependency_analyzer/cyclonedx/resources/apache_maven_root_sbom.json rename to tests/integration/cases/apache_maven_sbom/sbom.json diff --git a/tests/integration/cases/apache_maven_sbom/test.yaml b/tests/integration/cases/apache_maven_sbom/test.yaml new file mode 100644 index 000000000..91bb779e3 --- /dev/null +++ b/tests/integration/cases/apache_maven_sbom/test.yaml @@ -0,0 +1,29 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing using a CycloneDx SBOM with target repo path + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/org.apache.maven/maven@4.0.0-alpha-1-SNAPSHOT?type=pom + - -rp + - https://github.com/apache/maven + - -b + - master + - -d + - 3fc399318edef0d5ba593723a24fff64291d6f9b + sbom: sbom.json +- name: Compare dependency report + kind: compare + options: + kind: deps_report + result: output/reports/maven/org_apache_maven/maven/dependencies.json + expected: dependencies.json From 2ad1f5eabac3509af4de58e5a6087845d6fd6ad3 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 01:24:20 +1000 Subject: [PATCH 24/47] chore: convert case purl_of_nonexistent_artifact Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 8 -------- .../purl_of_nonexistent_artifact/policy.dl} | 0 .../purl_of_nonexistent_artifact/test.yaml | 20 +++++++++++++++++++ 3 files changed, 20 insertions(+), 8 deletions(-) rename tests/{e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.dl => integration/cases/purl_of_nonexistent_artifact/policy.dl} (100%) create mode 100644 tests/integration/cases/purl_of_nonexistent_artifact/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index f5c001fc4..b9fb96e32 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -207,14 +207,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "com.example/nonexistent: Analyzing purl of nonexistent artifact." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.dl -run_macaron_clean $ANALYZE -purl pkg:maven/com.example/nonexistent@1.0.0 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - # Analyze micronaut-projects/micronaut-test. echo -e "\n==================================================================================" echo "Run integration tests with configurations for micronaut-projects/micronaut-test..." diff --git a/tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.dl b/tests/integration/cases/purl_of_nonexistent_artifact/policy.dl similarity index 100% rename from tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.dl rename to tests/integration/cases/purl_of_nonexistent_artifact/policy.dl diff --git a/tests/integration/cases/purl_of_nonexistent_artifact/test.yaml b/tests/integration/cases/purl_of_nonexistent_artifact/test.yaml new file mode 100644 index 000000000..1b8d136a3 --- /dev/null +++ b/tests/integration/cases/purl_of_nonexistent_artifact/test.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing purl of nonexistent artifact. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/com.example/nonexistent@1.0.0 + - --skip-deps +- name: Run macaron verify-policy + kind: verify + options: + main_args: + - --verbose + policy: policy.dl From 16dd21a0aa187a2d87963cba8900d1c66a04cd36 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 02:13:02 +1000 Subject: [PATCH 25/47] chore: convert case sigstore_mock Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 8 ------ .../cases/sigstore_mock/policy.dl} | 0 .../integration/cases/sigstore_mock/test.yaml | 27 +++++++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) rename tests/{e2e/expected_results/purl/npm/sigstore/mock/mock.dl => integration/cases/sigstore_mock/policy.dl} (100%) create mode 100644 tests/integration/cases/sigstore_mock/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index b9fb96e32..767fc069b 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -96,14 +96,6 @@ run_macaron_clean -dp $DEFAULTS_FILE $ANALYZE -purl pkg:maven/io.micronaut/micro $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail if [[ -z "$NO_NPM_TEST" ]]; then - echo -e "\n----------------------------------------------------------------------------------" - echo "sigstore/mock@0.1.0: Analyzing the PURL when automatic dependency resolution is skipped." - echo -e "----------------------------------------------------------------------------------\n" - OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/purl/npm/sigstore/mock/mock.dl - run_macaron_clean $ANALYZE -purl pkg:npm/@sigstore/mock@0.1.0 -rp https://github.com/sigstore/sigstore-js -b main -d ebdcfdfbdfeb9c9aeee6df53674ef230613629f5 --skip-deps || log_fail - - $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "semver@7.6.0: Extracting repository URL and commit from provenance while Repo Finder is disabled." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/e2e/expected_results/purl/npm/sigstore/mock/mock.dl b/tests/integration/cases/sigstore_mock/policy.dl similarity index 100% rename from tests/e2e/expected_results/purl/npm/sigstore/mock/mock.dl rename to tests/integration/cases/sigstore_mock/policy.dl diff --git a/tests/integration/cases/sigstore_mock/test.yaml b/tests/integration/cases/sigstore_mock/test.yaml new file mode 100644 index 000000000..3305f3ecc --- /dev/null +++ b/tests/integration/cases/sigstore_mock/test.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the PURL when automatic dependency resolution is skipped. + +tags: +- npm-registry + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:npm/@sigstore/mock@0.1.0 + - -rp + - https://github.com/sigstore/sigstore-js + - -b + - main + - -d + - ebdcfdfbdfeb9c9aeee6df53674ef230613629f5 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 767f51a3a6e37762fa2a5843d42199eac00edb69 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 02:19:58 +1000 Subject: [PATCH 26/47] chore: convert case semver Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 10 --------- .../cases/semver/config.ini} | 0 .../cases/semver/policy.dl} | 0 tests/integration/cases/semver/test.yaml | 21 +++++++++++++++++++ 4 files changed, 21 insertions(+), 10 deletions(-) rename tests/{e2e/defaults/disable_repo_finder.ini => integration/cases/semver/config.ini} (100%) rename tests/{e2e/expected_results/purl/npm/semver/semver.dl => integration/cases/semver/policy.dl} (100%) create mode 100644 tests/integration/cases/semver/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 767fc069b..199c9541a 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -95,16 +95,6 @@ run_macaron_clean -dp $DEFAULTS_FILE $ANALYZE -purl pkg:maven/io.micronaut/micro $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -if [[ -z "$NO_NPM_TEST" ]]; then - echo -e "\n----------------------------------------------------------------------------------" - echo "semver@7.6.0: Extracting repository URL and commit from provenance while Repo Finder is disabled." - echo -e "----------------------------------------------------------------------------------\n" - OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/purl/npm/semver/semver.dl - run_macaron_clean -dp tests/e2e/defaults/disable_repo_finder.ini $ANALYZE -purl pkg:npm/semver@7.6.0 || log_fail - - $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -fi - echo -e "\n----------------------------------------------------------------------------------" echo "gitlab.com/tinyMediaManager/tinyMediaManager: Analyzing the repo path and the branch name when automatic dependency resolution is skipped." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/e2e/defaults/disable_repo_finder.ini b/tests/integration/cases/semver/config.ini similarity index 100% rename from tests/e2e/defaults/disable_repo_finder.ini rename to tests/integration/cases/semver/config.ini diff --git a/tests/e2e/expected_results/purl/npm/semver/semver.dl b/tests/integration/cases/semver/policy.dl similarity index 100% rename from tests/e2e/expected_results/purl/npm/semver/semver.dl rename to tests/integration/cases/semver/policy.dl diff --git a/tests/integration/cases/semver/test.yaml b/tests/integration/cases/semver/test.yaml new file mode 100644 index 000000000..a9f2c62d4 --- /dev/null +++ b/tests/integration/cases/semver/test.yaml @@ -0,0 +1,21 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Extracting repository URL and commit from provenance while Repo Finder is disabled. + +tags: +- npm-registry + +steps: +- name: Run macaron analyze + kind: analyze + options: + ini: config.ini + command_args: + - -purl + - pkg:npm/semver@7.6.0 +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 68af3d14a3adcc66c5a1b62f3380e5eca4b6218a Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 02:48:01 +1000 Subject: [PATCH 27/47] chore: convert case micronaut-projects_micronaut-core Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 21 ------------- .../check_results_policy.dl} | 0 .../config.ini} | 0 .../deploy_info_policy.dl} | 0 .../policy_report.json} | 0 .../test.yaml | 30 +++++++++++++++++++ 6 files changed, 30 insertions(+), 21 deletions(-) rename tests/{e2e/expected_results/purl/maven/micronaut-core/micronaut-core.dl => integration/cases/micronaut-projects_micronaut-core/check_results_policy.dl} (100%) rename tests/{e2e/defaults/micronaut-core.ini => integration/cases/micronaut-projects_micronaut-core/config.ini} (100%) rename tests/{policy_engine/resources/policies/micronaut-core/test_deploy_info.dl => integration/cases/micronaut-projects_micronaut-core/deploy_info_policy.dl} (100%) rename tests/{policy_engine/expected_results/micronaut-core/test_deploy_info.json => integration/cases/micronaut-projects_micronaut-core/policy_report.json} (100%) create mode 100644 tests/integration/cases/micronaut-projects_micronaut-core/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 199c9541a..c76d99e59 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -86,15 +86,6 @@ echo -e "\n===================================================================== echo "Run integration tests without configurations" echo -e "==================================================================================\n" -echo -e "\n----------------------------------------------------------------------------------" -echo "micronaut-projects/micronaut-core: Analyzing the PURL when automatic dependency resolution is skipped." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/purl/maven/micronaut-core/micronaut-core.dl -DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/micronaut-core.ini -run_macaron_clean -dp $DEFAULTS_FILE $ANALYZE -purl pkg:maven/io.micronaut/micronaut-core@4.2.3 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "gitlab.com/tinyMediaManager/tinyMediaManager: Analyzing the repo path and the branch name when automatic dependency resolution is skipped." echo -e "----------------------------------------------------------------------------------\n" @@ -547,18 +538,6 @@ run_macaron_clean $ANALYZE -pe $EXPECTATION_FILE -rp https://github.com/urllib3/ $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "Run policy CLI with micronaut-core results to test deploy command information." -echo -e "----------------------------------------------------------------------------------\n" -POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/micronaut-core/test_deploy_info.dl -POLICY_RESULT=$WORKSPACE/output/policy_report.json -POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/micronaut-core/test_deploy_info.json -DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/micronaut-core.ini -$RUN_MACARON -dp $DEFAULTS_FILE analyze -purl pkg:maven/io.micronaut/micronaut-core@4.2.3 --skip-deps || log_fail - -$RUN_POLICY -f $POLICY_FILE -d $DB || log_fail -check_or_update_expected_output $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail - # Testing the Repo Finder's remote calls. # This requires the 'packageurl' Python module echo -e "\n----------------------------------------------------------------------------------" diff --git a/tests/e2e/expected_results/purl/maven/micronaut-core/micronaut-core.dl b/tests/integration/cases/micronaut-projects_micronaut-core/check_results_policy.dl similarity index 100% rename from tests/e2e/expected_results/purl/maven/micronaut-core/micronaut-core.dl rename to tests/integration/cases/micronaut-projects_micronaut-core/check_results_policy.dl diff --git a/tests/e2e/defaults/micronaut-core.ini b/tests/integration/cases/micronaut-projects_micronaut-core/config.ini similarity index 100% rename from tests/e2e/defaults/micronaut-core.ini rename to tests/integration/cases/micronaut-projects_micronaut-core/config.ini diff --git a/tests/policy_engine/resources/policies/micronaut-core/test_deploy_info.dl b/tests/integration/cases/micronaut-projects_micronaut-core/deploy_info_policy.dl similarity index 100% rename from tests/policy_engine/resources/policies/micronaut-core/test_deploy_info.dl rename to tests/integration/cases/micronaut-projects_micronaut-core/deploy_info_policy.dl diff --git a/tests/policy_engine/expected_results/micronaut-core/test_deploy_info.json b/tests/integration/cases/micronaut-projects_micronaut-core/policy_report.json similarity index 100% rename from tests/policy_engine/expected_results/micronaut-core/test_deploy_info.json rename to tests/integration/cases/micronaut-projects_micronaut-core/policy_report.json diff --git a/tests/integration/cases/micronaut-projects_micronaut-core/test.yaml b/tests/integration/cases/micronaut-projects_micronaut-core/test.yaml new file mode 100644 index 000000000..7731bfed6 --- /dev/null +++ b/tests/integration/cases/micronaut-projects_micronaut-core/test.yaml @@ -0,0 +1,30 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the PURL when automatic dependency resolution is skipped. + Run policy CLI with micronaut-core results to test deploy command information. + +steps: +- name: Run macaron analyze + kind: analyze + options: + ini: config.ini + command_args: + - -purl + - pkg:maven/io.micronaut/micronaut-core@4.2.3 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: check_results_policy.dl +- name: Run macaron verify-policy to verify deploy command information + kind: verify + options: + policy: deploy_info_policy.dl +- name: Compare policy verification report. Run macaron verify-policy to verify deploy command information + kind: compare + options: + kind: policy_report + result: output/policy_report.json + expected: policy_report.json From 62ba583e572a259e05d67cb88dc892a37fa8c6de Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 03:11:01 +1000 Subject: [PATCH 28/47] chore: convert case gitlab_tinyMediaManager Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 8 ------- .../cases/gitlab_tinyMediaManager/policy.dl} | 0 .../cases/gitlab_tinyMediaManager/test.yaml | 22 +++++++++++++++++++ 3 files changed, 22 insertions(+), 8 deletions(-) rename tests/{e2e/expected_results/tinyMediaManager/tinyMediaManager.dl => integration/cases/gitlab_tinyMediaManager/policy.dl} (100%) create mode 100644 tests/integration/cases/gitlab_tinyMediaManager/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index c76d99e59..27b8ef3ce 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -86,14 +86,6 @@ echo -e "\n===================================================================== echo "Run integration tests without configurations" echo -e "==================================================================================\n" -echo -e "\n----------------------------------------------------------------------------------" -echo "gitlab.com/tinyMediaManager/tinyMediaManager: Analyzing the repo path and the branch name when automatic dependency resolution is skipped." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/tinyMediaManager/tinyMediaManager.dl -run_macaron_clean $ANALYZE -rp https://gitlab.com/tinyMediaManager/tinyMediaManager -b main -d cca6b67a335074eca42136556f0a321f75dc4f48 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "jenkinsci/plot-plugin: Analyzing the repo path, the branch name and the commit digest when automatic dependency resolution is skipped." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/e2e/expected_results/tinyMediaManager/tinyMediaManager.dl b/tests/integration/cases/gitlab_tinyMediaManager/policy.dl similarity index 100% rename from tests/e2e/expected_results/tinyMediaManager/tinyMediaManager.dl rename to tests/integration/cases/gitlab_tinyMediaManager/policy.dl diff --git a/tests/integration/cases/gitlab_tinyMediaManager/test.yaml b/tests/integration/cases/gitlab_tinyMediaManager/test.yaml new file mode 100644 index 000000000..6ab22ed33 --- /dev/null +++ b/tests/integration/cases/gitlab_tinyMediaManager/test.yaml @@ -0,0 +1,22 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the repo path and the branch name when automatic dependency resolution is skipped. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://gitlab.com/tinyMediaManager/tinyMediaManager + - -b + - main + - -d + - cca6b67a335074eca42136556f0a321f75dc4f48 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 6599f500687c01eb30815a0dc7f12fd31fdc7a3c Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 03:16:52 +1000 Subject: [PATCH 29/47] chore: convert case gitlab_tinyMediaManager_purl Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 9 ------- .../gitlab_tinyMediaManager_purl/policy.dl} | 0 .../gitlab_tinyMediaManager_purl/test.yaml | 27 +++++++++++++++++++ 3 files changed, 27 insertions(+), 9 deletions(-) rename tests/{e2e/expected_results/purl/org_tinymediamanager/tinyMediaManager.dl => integration/cases/gitlab_tinyMediaManager_purl/policy.dl} (100%) create mode 100644 tests/integration/cases/gitlab_tinyMediaManager_purl/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 27b8ef3ce..0e895fd69 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -283,15 +283,6 @@ run_macaron_clean $ANALYZE -purl pkg:maven/com.google.guava/guava@32.1.2-jre?typ $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "org.tinymediamanager/tinyMediaManager: Analyzing the purl with a version, and a provided repo with no commit." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/purl/org_tinymediamanager/tinyMediaManager.dl -run_macaron_clean $ANALYZE -purl pkg:maven/org.tinymediamanager/tinyMediaManager@4.3.13 -rp https://gitlab.com/tinyMediaManager/tinyMediaManager --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - - # Running Macaron using local paths. echo -e "\n==================================================================================" echo "Run integration tests with local paths for apache/maven..." diff --git a/tests/e2e/expected_results/purl/org_tinymediamanager/tinyMediaManager.dl b/tests/integration/cases/gitlab_tinyMediaManager_purl/policy.dl similarity index 100% rename from tests/e2e/expected_results/purl/org_tinymediamanager/tinyMediaManager.dl rename to tests/integration/cases/gitlab_tinyMediaManager_purl/policy.dl diff --git a/tests/integration/cases/gitlab_tinyMediaManager_purl/test.yaml b/tests/integration/cases/gitlab_tinyMediaManager_purl/test.yaml new file mode 100644 index 000000000..162197b24 --- /dev/null +++ b/tests/integration/cases/gitlab_tinyMediaManager_purl/test.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the purl with a version, and a provided repo with no commit. + +tags: +- npm-registry + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/org.tinymediamanager/tinyMediaManager@4.3.13 + - -rp + - https://gitlab.com/tinyMediaManager/tinyMediaManager + - -b + - main + - -d + - cca6b67a335074eca42136556f0a321f75dc4f48 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 5caa3a3b2ddb40e624fb5ca3197f2cee23cbbb50 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 03:31:07 +1000 Subject: [PATCH 30/47] chore: convert case jenkinsci_plotplugin Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 8 ------- .../cases/jenkinsci_plotplugin/policy.dl} | 0 .../cases/jenkinsci_plotplugin/test.yaml | 22 +++++++++++++++++++ 3 files changed, 22 insertions(+), 8 deletions(-) rename tests/{e2e/expected_results/plot-plugin/plot-plugin.dl => integration/cases/jenkinsci_plotplugin/policy.dl} (100%) create mode 100644 tests/integration/cases/jenkinsci_plotplugin/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 0e895fd69..83a1c0508 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -86,14 +86,6 @@ echo -e "\n===================================================================== echo "Run integration tests without configurations" echo -e "==================================================================================\n" -echo -e "\n----------------------------------------------------------------------------------" -echo "jenkinsci/plot-plugin: Analyzing the repo path, the branch name and the commit digest when automatic dependency resolution is skipped." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/plot-plugin/plot-plugin.dl -run_macaron_clean $ANALYZE -rp https://github.com/jenkinsci/plot-plugin -b master -d 55b059187e252b35ac0d6cb52268833ee1bb7380 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "timyarkov/multibuild_test: Analyzing Maven artifact with the repo path, the branch name and the commit digest" echo "with dependency resolution using cyclonedx Maven plugins (defaults)." diff --git a/tests/e2e/expected_results/plot-plugin/plot-plugin.dl b/tests/integration/cases/jenkinsci_plotplugin/policy.dl similarity index 100% rename from tests/e2e/expected_results/plot-plugin/plot-plugin.dl rename to tests/integration/cases/jenkinsci_plotplugin/policy.dl diff --git a/tests/integration/cases/jenkinsci_plotplugin/test.yaml b/tests/integration/cases/jenkinsci_plotplugin/test.yaml new file mode 100644 index 000000000..def7c98ab --- /dev/null +++ b/tests/integration/cases/jenkinsci_plotplugin/test.yaml @@ -0,0 +1,22 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the repo path, the branch name and the commit digest when automatic dependency resolution is skipped. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://github.com/jenkinsci/plot-plugin + - -b + - master + - -d + - 55b059187e252b35ac0d6cb52268833ee1bb7380 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From a2f1bc885df4ecce1fd0ea382a18d59bba6ea753 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 03:45:17 +1000 Subject: [PATCH 31/47] chore: convert case timyarkov_docker_test Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 8 ------- .../cases/timyarkov_docker_test/policy.dl} | 0 .../cases/timyarkov_docker_test/test.yaml | 23 +++++++++++++++++++ 3 files changed, 23 insertions(+), 8 deletions(-) rename tests/{e2e/expected_results/docker_test/docker_test.dl => integration/cases/timyarkov_docker_test/policy.dl} (100%) create mode 100644 tests/integration/cases/timyarkov_docker_test/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 83a1c0508..53645c2ea 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -99,14 +99,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo "timyarkov/docker_test: Analyzing the repo path, the branch name and the commit digest" -echo "when automatic dependency resolution is skipped, for a project using docker as a build tool." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/docker_test/docker_test.dl -run_macaron_clean $ANALYZE -rp https://github.com/timyarkov/docker_test -b main -d 404a51a2f38c4470af6b32e4e00b5318c2d7c0cc --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "uiv-lib/uiv: Analysing the repo path, the branch name and the commit digest for an npm project," echo "skipping dependency resolution." diff --git a/tests/e2e/expected_results/docker_test/docker_test.dl b/tests/integration/cases/timyarkov_docker_test/policy.dl similarity index 100% rename from tests/e2e/expected_results/docker_test/docker_test.dl rename to tests/integration/cases/timyarkov_docker_test/policy.dl diff --git a/tests/integration/cases/timyarkov_docker_test/test.yaml b/tests/integration/cases/timyarkov_docker_test/test.yaml new file mode 100644 index 000000000..4a0f2a15e --- /dev/null +++ b/tests/integration/cases/timyarkov_docker_test/test.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the repo path, the branch name and the commit digest when automatic dependency resolution + is skipped, for a project using docker as a build tool. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://github.com/timyarkov/docker_test + - -b + - main + - -d + - 404a51a2f38c4470af6b32e4e00b5318c2d7c0cc + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 622f5027cf302ef452185446fc033690e764ad49 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 03:49:01 +1000 Subject: [PATCH 32/47] chore: convert case uiv-lib_uiv Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 9 -------- .../cases/uiv-lib_uiv/policy.dl} | 0 tests/integration/cases/uiv-lib_uiv/test.yaml | 23 +++++++++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) rename tests/{e2e/expected_results/uiv/uiv.dl => integration/cases/uiv-lib_uiv/policy.dl} (100%) create mode 100644 tests/integration/cases/uiv-lib_uiv/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 53645c2ea..84f4e5095 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -99,15 +99,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "uiv-lib/uiv: Analysing the repo path, the branch name and the commit digest for an npm project," -echo "skipping dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/uiv/uiv.dl -run_macaron_clean $ANALYZE -rp https://github.com/uiv-lib/uiv -b dev -d 057b25b4db0913edab4cf728c306085e6fc20d49 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "onu-ui/onu-ui: Analysing the repo path, the branch name and the commit digest for a pnpm project," echo "skipping dependency resolution." diff --git a/tests/e2e/expected_results/uiv/uiv.dl b/tests/integration/cases/uiv-lib_uiv/policy.dl similarity index 100% rename from tests/e2e/expected_results/uiv/uiv.dl rename to tests/integration/cases/uiv-lib_uiv/policy.dl diff --git a/tests/integration/cases/uiv-lib_uiv/test.yaml b/tests/integration/cases/uiv-lib_uiv/test.yaml new file mode 100644 index 000000000..3bed519dd --- /dev/null +++ b/tests/integration/cases/uiv-lib_uiv/test.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analysing the repo path, the branch name and the commit digest for an npm project, + skipping dependency resolution. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://github.com/uiv-lib/uiv + - -b + - dev + - -d + - 057b25b4db0913edab4cf728c306085e6fc20d49 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From c69a77d1a5f271a1bd251896e1216d2f97499ee5 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 03:55:13 +1000 Subject: [PATCH 33/47] chore: convert case onu-ui_onu-ui_pnpm Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 9 -------- .../cases/onu-ui_onu-ui_pnpm/policy.dl} | 0 .../cases/onu-ui_onu-ui_pnpm/test.yaml | 23 +++++++++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) rename tests/{e2e/expected_results/onu-ui/onu-ui.dl => integration/cases/onu-ui_onu-ui_pnpm/policy.dl} (100%) create mode 100644 tests/integration/cases/onu-ui_onu-ui_pnpm/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 84f4e5095..aaf14b37e 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -99,15 +99,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "onu-ui/onu-ui: Analysing the repo path, the branch name and the commit digest for a pnpm project," -echo "skipping dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/onu-ui/onu-ui.dl -run_macaron_clean $ANALYZE -rp https://github.com/onu-ui/onu-ui -b main -d e3f2825c3940002a920d65476116a64684b3d95e --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "facebook/yoga: Analysing the repo path, the branch name and the commit digest for a Yarn classic" echo "project, skipping dependency resolution." diff --git a/tests/e2e/expected_results/onu-ui/onu-ui.dl b/tests/integration/cases/onu-ui_onu-ui_pnpm/policy.dl similarity index 100% rename from tests/e2e/expected_results/onu-ui/onu-ui.dl rename to tests/integration/cases/onu-ui_onu-ui_pnpm/policy.dl diff --git a/tests/integration/cases/onu-ui_onu-ui_pnpm/test.yaml b/tests/integration/cases/onu-ui_onu-ui_pnpm/test.yaml new file mode 100644 index 000000000..c41d598f4 --- /dev/null +++ b/tests/integration/cases/onu-ui_onu-ui_pnpm/test.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analysing the repo path, the branch name and the commit digest for a pnpm project, + skipping dependency resolution. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://github.com/onu-ui/onu-ui + - -b + - main + - -d + - e3f2825c3940002a920d65476116a64684b3d95e + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 07568170e7412331eca68de9aa4ef353a3966891 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 03:58:53 +1000 Subject: [PATCH 34/47] chore: convert case facebook_yoga_yarn_classic Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 9 -------- .../facebook_yoga_yarn_classic/policy.dl} | 0 .../facebook_yoga_yarn_classic/test.yaml | 23 +++++++++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) rename tests/{e2e/expected_results/yoga/yoga.dl => integration/cases/facebook_yoga_yarn_classic/policy.dl} (100%) create mode 100644 tests/integration/cases/facebook_yoga_yarn_classic/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index aaf14b37e..baf652e85 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -99,15 +99,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "facebook/yoga: Analysing the repo path, the branch name and the commit digest for a Yarn classic" -echo "project, skipping dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/yoga/yoga.dl -run_macaron_clean $ANALYZE -rp https://github.com/facebook/yoga -b main -d f8e2bc0875c145c429d0e865c9b83a40f65b3070 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "wojtekmaj/react-pdf: Analysing the repo path, the branch name and the commit digest for a Yarn modern" echo "project, skipping dependency resolution." diff --git a/tests/e2e/expected_results/yoga/yoga.dl b/tests/integration/cases/facebook_yoga_yarn_classic/policy.dl similarity index 100% rename from tests/e2e/expected_results/yoga/yoga.dl rename to tests/integration/cases/facebook_yoga_yarn_classic/policy.dl diff --git a/tests/integration/cases/facebook_yoga_yarn_classic/test.yaml b/tests/integration/cases/facebook_yoga_yarn_classic/test.yaml new file mode 100644 index 000000000..225742eba --- /dev/null +++ b/tests/integration/cases/facebook_yoga_yarn_classic/test.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analysing the repo path, the branch name and the commit digest for a Yarn classic + project, skipping dependency resolution. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://github.com/facebook/yoga + - -b + - main + - -d + - f8e2bc0875c145c429d0e865c9b83a40f65b3070 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 803f2bd8a4299cc09ebc346ec30dd911ff8187dc Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 04:03:32 +1000 Subject: [PATCH 35/47] chore: convert case wojtekmaj_reactpdf_yarn_modern Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 9 -------- .../wojtekmaj_reactpdf_yarn_modern/policy.dl} | 0 .../wojtekmaj_reactpdf_yarn_modern/test.yaml | 23 +++++++++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) rename tests/{e2e/expected_results/react-pdf/react-pdf.dl => integration/cases/wojtekmaj_reactpdf_yarn_modern/policy.dl} (100%) create mode 100644 tests/integration/cases/wojtekmaj_reactpdf_yarn_modern/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index baf652e85..5e4ac4b03 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -99,15 +99,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "wojtekmaj/react-pdf: Analysing the repo path, the branch name and the commit digest for a Yarn modern" -echo "project, skipping dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/react-pdf/react-pdf.dl -run_macaron_clean $ANALYZE -rp https://github.com/wojtekmaj/react-pdf -b main -d be18436b7be827eb993b2e1e4bd9230dd835a9a3 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "sigstore/sget: Analysing the repo path, the branch name and the" echo "commit digest for a Go project, skipping dependency resolution." diff --git a/tests/e2e/expected_results/react-pdf/react-pdf.dl b/tests/integration/cases/wojtekmaj_reactpdf_yarn_modern/policy.dl similarity index 100% rename from tests/e2e/expected_results/react-pdf/react-pdf.dl rename to tests/integration/cases/wojtekmaj_reactpdf_yarn_modern/policy.dl diff --git a/tests/integration/cases/wojtekmaj_reactpdf_yarn_modern/test.yaml b/tests/integration/cases/wojtekmaj_reactpdf_yarn_modern/test.yaml new file mode 100644 index 000000000..ae84ebb33 --- /dev/null +++ b/tests/integration/cases/wojtekmaj_reactpdf_yarn_modern/test.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analysing the repo path, the branch name and the commit digest for a Yarn modern + project, skipping dependency resolution. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://github.com/wojtekmaj/react-pdf + - -b + - main + - -d + - be18436b7be827eb993b2e1e4bd9230dd835a9a3 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From e4440c301df812196ae7557dd7f0c97a8038a2d5 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 04:05:51 +1000 Subject: [PATCH 36/47] chore: convert case sigstore_sget Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 9 -------- .../cases/sigstore_sget/policy.dl} | 0 .../integration/cases/sigstore_sget/test.yaml | 23 +++++++++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) rename tests/{e2e/expected_results/sget/sget.dl => integration/cases/sigstore_sget/policy.dl} (100%) create mode 100644 tests/integration/cases/sigstore_sget/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 5e4ac4b03..5746ff88a 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -99,15 +99,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "sigstore/sget: Analysing the repo path, the branch name and the" -echo "commit digest for a Go project, skipping dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/sget/sget.dl -run_macaron_clean $ANALYZE -rp https://github.com/sigstore/sget -b main -d 99e7b91204d391ccc76507f7079b6d2a7957489e --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Analyzing the repo path, the branch name and the commit digest with dependency resolution using cyclonedx maven plugin (default)." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/e2e/expected_results/sget/sget.dl b/tests/integration/cases/sigstore_sget/policy.dl similarity index 100% rename from tests/e2e/expected_results/sget/sget.dl rename to tests/integration/cases/sigstore_sget/policy.dl diff --git a/tests/integration/cases/sigstore_sget/test.yaml b/tests/integration/cases/sigstore_sget/test.yaml new file mode 100644 index 000000000..2919ffefe --- /dev/null +++ b/tests/integration/cases/sigstore_sget/test.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analysing the repo path, the branch name and the + commit digest for a Go project, skipping dependency resolution. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://github.com/sigstore/sget + - -b + - main + - -d + - 99e7b91204d391ccc76507f7079b6d2a7957489e + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From c38ce6eb219a87146f229c04f1a78f3efed58f34 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 15:54:37 +1000 Subject: [PATCH 37/47] chore: convert case google_guava Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 8 -------- .../cases/google_guava/policy.dl} | 0 tests/integration/cases/google_guava/test.yaml | 18 ++++++++++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) rename tests/{e2e/expected_results/purl/com_google_guava/guava/guava.dl => integration/cases/google_guava/policy.dl} (100%) create mode 100644 tests/integration/cases/google_guava/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 5746ff88a..50723c2c3 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -214,14 +214,6 @@ $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail # check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "google/guava: Analyzing with PURL and repository path without dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/purl/com_google_guava/guava/guava.dl -run_macaron_clean $ANALYZE -purl pkg:maven/com.google.guava/guava@32.1.2-jre?type=jar --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - # Running Macaron using local paths. echo -e "\n==================================================================================" echo "Run integration tests with local paths for apache/maven..." diff --git a/tests/e2e/expected_results/purl/com_google_guava/guava/guava.dl b/tests/integration/cases/google_guava/policy.dl similarity index 100% rename from tests/e2e/expected_results/purl/com_google_guava/guava/guava.dl rename to tests/integration/cases/google_guava/policy.dl diff --git a/tests/integration/cases/google_guava/test.yaml b/tests/integration/cases/google_guava/test.yaml new file mode 100644 index 000000000..8f2f0e97e --- /dev/null +++ b/tests/integration/cases/google_guava/test.yaml @@ -0,0 +1,18 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing with PURL and repository path without dependency resolution. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/com.google.guava/guava@32.1.2-jre?type=jar + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 9d31a8730e017aaa31170300773170ccbf08aae8 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 16:32:48 +1000 Subject: [PATCH 38/47] chore: convert case snakeyaml_unsupported_git_service Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 10 ------ .../config.ini} | 0 .../policy.dl} | 0 .../test.yaml | 32 +++++++++++++++++++ 4 files changed, 32 insertions(+), 10 deletions(-) rename tests/{e2e/defaults/bitbucket_local_repo.ini => integration/cases/snakeyaml_unsupported_git_service/config.ini} (100%) rename tests/{e2e/expected_results/snakeyaml/snakeyaml.dl => integration/cases/snakeyaml_unsupported_git_service/policy.dl} (100%) create mode 100644 tests/integration/cases/snakeyaml_unsupported_git_service/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 50723c2c3..ee0f124f2 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -219,16 +219,6 @@ echo -e "\n===================================================================== echo "Run integration tests with local paths for apache/maven..." echo -e "==================================================================================\n" -echo -e "\n----------------------------------------------------------------------------------" -echo "bitbucket.org/snakeyaml/snakeyaml: Analyzing a repository with un-supported git service as local repo without dependency resolution." -echo -e "----------------------------------------------------------------------------------\n" -git clone https://bitbucket.org/snakeyaml/snakeyaml $WORKSPACE/output/local_repos/snakeyaml || log_fail -DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/bitbucket_local_repo.ini -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/snakeyaml/snakeyaml.dl -run_macaron_clean -dp $DEFAULTS_FILE -lr $WORKSPACE/output/local_repos $ANALYZE -rp snakeyaml -d a34989252e6f59e36a3aaf788a903b7a37a73d33 --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Analyzing local clone with the branch name, the commit digest and dependency resolution using cyclonedx maven plugin (default)." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/e2e/defaults/bitbucket_local_repo.ini b/tests/integration/cases/snakeyaml_unsupported_git_service/config.ini similarity index 100% rename from tests/e2e/defaults/bitbucket_local_repo.ini rename to tests/integration/cases/snakeyaml_unsupported_git_service/config.ini diff --git a/tests/e2e/expected_results/snakeyaml/snakeyaml.dl b/tests/integration/cases/snakeyaml_unsupported_git_service/policy.dl similarity index 100% rename from tests/e2e/expected_results/snakeyaml/snakeyaml.dl rename to tests/integration/cases/snakeyaml_unsupported_git_service/policy.dl diff --git a/tests/integration/cases/snakeyaml_unsupported_git_service/test.yaml b/tests/integration/cases/snakeyaml_unsupported_git_service/test.yaml new file mode 100644 index 000000000..655fd2dbc --- /dev/null +++ b/tests/integration/cases/snakeyaml_unsupported_git_service/test.yaml @@ -0,0 +1,32 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing a repository with un-supported git service as local repo without dependency resolution. + +steps: +- name: Prepare local repo directory + kind: shell + options: + cmd: mkdir -p output/local_repos +- name: Clone snakeyaml + kind: shell + options: + cmd: git clone --filter=tree:0 https://bitbucket.org/snakeyaml/snakeyaml output/local_repos/snakeyaml +- name: Run macaron analyze + kind: analyze + options: + ini: config.ini + main_args: + - -lr + - output/local_repos + command_args: + - -rp + - snakeyaml + - -d + - a34989252e6f59e36a3aaf788a903b7a37a73d33 + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 821e202659ffad67d2a68143e82bab0a70cfe1f6 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 16:41:52 +1000 Subject: [PATCH 39/47] chore: convert case all_checks_excluded Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 11 ----------- .../cases/all_checks_excluded/config.ini} | 0 .../cases/all_checks_excluded/test.yaml | 16 ++++++++++++++++ 3 files changed, 16 insertions(+), 11 deletions(-) rename tests/{e2e/defaults/exclude_all_checks.ini => integration/cases/all_checks_excluded/config.ini} (100%) create mode 100644 tests/integration/cases/all_checks_excluded/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index ee0f124f2..b50c50a43 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -360,17 +360,6 @@ fi rm -rf "$SOURCE_REPO" rm -rf "$TARGET_REPO" -echo -e "\n----------------------------------------------------------------------------------" -echo "Running the analysis with all checks excluded. This test should return an error code." -echo -e "----------------------------------------------------------------------------------\n" -run_macaron_clean -dp tests/e2e/defaults/exclude_all_checks.ini $ANALYZE -rp https://github.com/apache/maven --skip-deps - -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi - echo -e "\n-----------------------------------------------------------------------------------------" echo "pkg:pypi/django@5.0.6: Analyzing the dependencies with an invalid path to the virtual env dir." echo -e "-----------------------------------------------------------------------------------------\n" diff --git a/tests/e2e/defaults/exclude_all_checks.ini b/tests/integration/cases/all_checks_excluded/config.ini similarity index 100% rename from tests/e2e/defaults/exclude_all_checks.ini rename to tests/integration/cases/all_checks_excluded/config.ini diff --git a/tests/integration/cases/all_checks_excluded/test.yaml b/tests/integration/cases/all_checks_excluded/test.yaml new file mode 100644 index 000000000..f81dc198c --- /dev/null +++ b/tests/integration/cases/all_checks_excluded/test.yaml @@ -0,0 +1,16 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Running the analysis with all checks excluded. This test should return an error code. + +steps: +- name: Run macaron analyze + kind: analyze + options: + ini: config.ini + command_args: + - -rp + - https://github.com/apache/maven + - --skip-deps + expect_fail: true From 7cde0365c16d0c149d7a70b11af2261e8a54f60b Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 17:32:53 +1000 Subject: [PATCH 40/47] chore: convert case missing_template_file Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 11 ----------- .../cases/missing_template_file/test.yaml | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 tests/integration/cases/missing_template_file/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index b50c50a43..37d6cf0e9 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -377,17 +377,6 @@ echo -e "----------------------------------------------------------------------- run_macaron_clean $ANALYZE -purl pkg:maven/private.apache.maven/maven@4.0.0-alpha-1-SNAPSHOT?type=pom || log_fail # We expect the analysis to finish with no errors. -echo -e "\n----------------------------------------------------------------------------------" -echo "Test using a custom template file that does not exist." -echo -e "----------------------------------------------------------------------------------\n" -run_macaron_clean $ANALYZE -rp https://github.com/apache/maven --skip-deps -g $WORKSPACE/should/not/exist - -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi - echo -e "\n----------------------------------------------------------------------------------" echo "Test providing an invalid provenance file as input." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/integration/cases/missing_template_file/test.yaml b/tests/integration/cases/missing_template_file/test.yaml new file mode 100644 index 000000000..3bb0fb515 --- /dev/null +++ b/tests/integration/cases/missing_template_file/test.yaml @@ -0,0 +1,17 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Test using a custom template file that does not exist. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -rp + - https://github.com/apache/maven + - --skip-deps + - -g + - does/not/exist + expect_fail: true From 42bea4868fe81bc04b709e832c2b50d8089be700 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 16:59:16 +1000 Subject: [PATCH 41/47] chore: convert case invalid_provenance_file Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 11 ----------- .../invalid_provenance.json | 1 + .../cases/invalid_provenance_file/test.yaml | 16 ++++++++++++++++ 3 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 tests/integration/cases/invalid_provenance_file/invalid_provenance.json create mode 100644 tests/integration/cases/invalid_provenance_file/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 37d6cf0e9..1ebee38ab 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -377,17 +377,6 @@ echo -e "----------------------------------------------------------------------- run_macaron_clean $ANALYZE -purl pkg:maven/private.apache.maven/maven@4.0.0-alpha-1-SNAPSHOT?type=pom || log_fail # We expect the analysis to finish with no errors. -echo -e "\n----------------------------------------------------------------------------------" -echo "Test providing an invalid provenance file as input." -echo -e "----------------------------------------------------------------------------------\n" -run_macaron_clean $ANALYZE -rp https://github.com/apache/maven --provenance-file $WORKSPACE/golang/internal/cue_validator/resources/invalid_provenance.json --skip-deps - -if [ $? -eq 0 ]; -then - echo -e "Expect non-zero status code but got $?." - log_fail -fi - echo -e "\n----------------------------------------------------------------------------------" echo "Test verifying CUE provenance expectation for slsa-verifier with explicitly-provided provenance file" echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/integration/cases/invalid_provenance_file/invalid_provenance.json b/tests/integration/cases/invalid_provenance_file/invalid_provenance.json new file mode 100644 index 000000000..b0047fa49 --- /dev/null +++ b/tests/integration/cases/invalid_provenance_file/invalid_provenance.json @@ -0,0 +1 @@ +None diff --git a/tests/integration/cases/invalid_provenance_file/test.yaml b/tests/integration/cases/invalid_provenance_file/test.yaml new file mode 100644 index 000000000..2fc291678 --- /dev/null +++ b/tests/integration/cases/invalid_provenance_file/test.yaml @@ -0,0 +1,16 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Test providing an invalid provenance file as input. + +steps: +- name: Run macaron analyze + kind: analyze + options: + provenance: invalid_provenance.json + command_args: + - -rp + - https://github.com/apache/maven + - --skip-deps + expect_fail: true From 2d60c019a63849c984c79ecb52f68501f0732f8d Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 17:50:53 +1000 Subject: [PATCH 42/47] chore: convert case urllib3_invalid_expectation Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 10 ------- .../invalid_expectation.cue | 1 + .../urllib3_invalid_expectation/policy.dl} | 0 .../urllib3_invalid_expectation/test.yaml | 27 +++++++++++++++++++ 4 files changed, 28 insertions(+), 10 deletions(-) create mode 100644 tests/integration/cases/urllib3_invalid_expectation/invalid_expectation.cue rename tests/{e2e/expected_results/urllib3/urllib3_cue_invalid.dl => integration/cases/urllib3_invalid_expectation/policy.dl} (100%) create mode 100644 tests/integration/cases/urllib3_invalid_expectation/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 1ebee38ab..4d3a11c4e 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -399,16 +399,6 @@ run_macaron_clean -dp $DEFAULTS_FILE $ANALYZE -pe $EXPECTATION_FILE -pf $PROVENA $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped" -echo "and CUE file is provided as expectation." -echo -e "----------------------------------------------------------------------------------\n" -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3_cue_invalid.dl -EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/invalid_expectations/invalid.cue -run_macaron_clean $ANALYZE -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - # Testing the Repo Finder's remote calls. # This requires the 'packageurl' Python module echo -e "\n----------------------------------------------------------------------------------" diff --git a/tests/integration/cases/urllib3_invalid_expectation/invalid_expectation.cue b/tests/integration/cases/urllib3_invalid_expectation/invalid_expectation.cue new file mode 100644 index 000000000..e420fe4d2 --- /dev/null +++ b/tests/integration/cases/urllib3_invalid_expectation/invalid_expectation.cue @@ -0,0 +1 @@ +INVALID diff --git a/tests/e2e/expected_results/urllib3/urllib3_cue_invalid.dl b/tests/integration/cases/urllib3_invalid_expectation/policy.dl similarity index 100% rename from tests/e2e/expected_results/urllib3/urllib3_cue_invalid.dl rename to tests/integration/cases/urllib3_invalid_expectation/policy.dl diff --git a/tests/integration/cases/urllib3_invalid_expectation/test.yaml b/tests/integration/cases/urllib3_invalid_expectation/test.yaml new file mode 100644 index 000000000..ffcb9a4b7 --- /dev/null +++ b/tests/integration/cases/urllib3_invalid_expectation/test.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing the repo path when automatic dependency resolution is skipped. + The CUE expectation file is invalid. + +tags: +- docker + +steps: +- name: Run macaron analyze with invalid expectation file + kind: analyze + options: + expectation: invalid_expectation.cue + command_args: + - --repo-path + - https://github.com/urllib3/urllib3/urllib3 + - --branch + - main + - --digest + - 87a0ecee6e691fe5ff93cd000c0158deebef763b + - --skip-deps +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From 03aa54cc02f7f5c97a3ec2c447fb8f79a134d571 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 18:22:16 +1000 Subject: [PATCH 43/47] chore: convert case micronaut-projects_micronaut-test Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 30 ----------------- .../caffeine.dl | 0 .../dependencies.json} | 0 .../micronaut-test.dl | 0 .../micronaut_test_config.yaml | 2 +- .../slf4j.dl | 0 .../test.yaml | 32 +++++++++++++++++++ 7 files changed, 33 insertions(+), 31 deletions(-) rename tests/{e2e/expected_results/micronaut-test => integration/cases/micronaut-projects_micronaut-test}/caffeine.dl (100%) rename tests/{dependency_analyzer/expected_results/skipdep_micronaut-projects_micronaut-test.json => integration/cases/micronaut-projects_micronaut-test/dependencies.json} (100%) rename tests/{e2e/expected_results/micronaut-test => integration/cases/micronaut-projects_micronaut-test}/micronaut-test.dl (100%) rename tests/{dependency_analyzer/configurations => integration/cases/micronaut-projects_micronaut-test}/micronaut_test_config.yaml (92%) rename tests/{e2e/expected_results/micronaut-test => integration/cases/micronaut-projects_micronaut-test}/slf4j.dl (100%) create mode 100644 tests/integration/cases/micronaut-projects_micronaut-test/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 4d3a11c4e..814310657 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -111,36 +111,6 @@ check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_f $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail -# Analyze micronaut-projects/micronaut-test. -echo -e "\n==================================================================================" -echo "Run integration tests with configurations for micronaut-projects/micronaut-test..." -echo -e "==================================================================================\n" -DEP_RESULT=$WORKSPACE/output/reports/github_com/micronaut-projects/micronaut-test/dependencies.json - -echo -e "\n----------------------------------------------------------------------------------" -echo "micronaut-projects/micronaut-test: Check the resolved dependency output when automatic dependency resolution is skipped." -echo -e "----------------------------------------------------------------------------------\n" -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/skipdep_micronaut-projects_micronaut-test.json -run_macaron_clean $ANALYZE -c $WORKSPACE/tests/dependency_analyzer/configurations/micronaut_test_config.yaml --skip-deps || log_fail - -check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - -echo -e "\n----------------------------------------------------------------------------------" -echo "micronaut-projects/micronaut-test: Check the e2e output JSON file with config when automatic dependency resolution is skipped." -echo -e "----------------------------------------------------------------------------------\n" -EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/micronaut-test - -declare -a COMPARE_FILES=( - "micronaut-test.dl" - "caffeine.dl" - "slf4j.dl" -) - -for i in "${COMPARE_FILES[@]}" -do - $RUN_POLICY -d $DB -f $EXPECT_DIR/$i || log_fail -done - # TODO: uncomment the test below after resolving https://github.com/oracle/macaron/issues/60. # echo -e "\n----------------------------------------------------------------------------------" # echo "micronaut-projects/micronaut-test: Check the resolved dependency output with config for cyclonedx gradle plugin (default)." diff --git a/tests/e2e/expected_results/micronaut-test/caffeine.dl b/tests/integration/cases/micronaut-projects_micronaut-test/caffeine.dl similarity index 100% rename from tests/e2e/expected_results/micronaut-test/caffeine.dl rename to tests/integration/cases/micronaut-projects_micronaut-test/caffeine.dl diff --git a/tests/dependency_analyzer/expected_results/skipdep_micronaut-projects_micronaut-test.json b/tests/integration/cases/micronaut-projects_micronaut-test/dependencies.json similarity index 100% rename from tests/dependency_analyzer/expected_results/skipdep_micronaut-projects_micronaut-test.json rename to tests/integration/cases/micronaut-projects_micronaut-test/dependencies.json diff --git a/tests/e2e/expected_results/micronaut-test/micronaut-test.dl b/tests/integration/cases/micronaut-projects_micronaut-test/micronaut-test.dl similarity index 100% rename from tests/e2e/expected_results/micronaut-test/micronaut-test.dl rename to tests/integration/cases/micronaut-projects_micronaut-test/micronaut-test.dl diff --git a/tests/dependency_analyzer/configurations/micronaut_test_config.yaml b/tests/integration/cases/micronaut-projects_micronaut-test/micronaut_test_config.yaml similarity index 92% rename from tests/dependency_analyzer/configurations/micronaut_test_config.yaml rename to tests/integration/cases/micronaut-projects_micronaut-test/micronaut_test_config.yaml index 780a01f05..4f3d254f1 100644 --- a/tests/dependency_analyzer/configurations/micronaut_test_config.yaml +++ b/tests/integration/cases/micronaut-projects_micronaut-test/micronaut_test_config.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: diff --git a/tests/e2e/expected_results/micronaut-test/slf4j.dl b/tests/integration/cases/micronaut-projects_micronaut-test/slf4j.dl similarity index 100% rename from tests/e2e/expected_results/micronaut-test/slf4j.dl rename to tests/integration/cases/micronaut-projects_micronaut-test/slf4j.dl diff --git a/tests/integration/cases/micronaut-projects_micronaut-test/test.yaml b/tests/integration/cases/micronaut-projects_micronaut-test/test.yaml new file mode 100644 index 000000000..aea7bd60f --- /dev/null +++ b/tests/integration/cases/micronaut-projects_micronaut-test/test.yaml @@ -0,0 +1,32 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Test configurations for micronaut-projects/micronaut-test. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -c + - micronaut_test_config.yaml + - --skip-deps +- name: Compare dependency report + kind: compare + options: + kind: deps_report + result: output/reports/github_com/micronaut-projects/micronaut-test/dependencies.json + expected: dependencies.json +- name: Run macaron verify-policy for micronaut-test + kind: verify + options: + policy: micronaut-test.dl +- name: Run macaron verify-policy for caffeine + kind: verify + options: + policy: caffeine.dl +- name: Run macaron verify-policy for slf4j + kind: verify + options: + policy: slf4j.dl From 616e3187952c0918037d2fab3c7a1627b12fd541 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 18:46:00 +1000 Subject: [PATCH 44/47] chore: convert case invalid_branch_or_commit_yaml_input Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 19 ----------------- .../maven_invalid_branch.yaml | 2 +- .../maven_invalid_commit.yaml} | 2 +- .../test.yaml | 21 +++++++++++++++++++ 4 files changed, 23 insertions(+), 21 deletions(-) rename tests/{e2e/configurations => integration/cases/invalid_branch_or_commit_yaml_input}/maven_invalid_branch.yaml (79%) rename tests/{e2e/configurations/maven_invalid_digest.yaml => integration/cases/invalid_branch_or_commit_yaml_input/maven_invalid_commit.yaml} (78%) create mode 100644 tests/integration/cases/invalid_branch_or_commit_yaml_input/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 814310657..ed1de2db5 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -134,25 +134,6 @@ run_macaron_clean $ANALYZE -c $WORKSPACE/tests/dependency_analyzer/configuration check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Check: Check the e2e status code of running with invalid branch or digest defined in the yaml configuration." -echo -e "----------------------------------------------------------------------------------\n" -declare -a INVALID_BRANCH_DIGEST=( - "maven_invalid_branch.yaml" - "maven_invalid_digest.yaml" -) - -for i in "${INVALID_BRANCH_DIGEST[@]}" -do - echo -e "Running with $WORKSPACE/tests/e2e/configurations/$i" - run_macaron_clean $ANALYZE -c $WORKSPACE/tests/e2e/configurations/$i - if [ $? -eq 0 ]; - then - echo -e "Expect non-zero status code for $WORKSPACE/test/e2e/configurations/$i but got $?." - log_fail - fi -done - echo -e "\n----------------------------------------------------------------------------------" echo "Test using the default template file." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/e2e/configurations/maven_invalid_branch.yaml b/tests/integration/cases/invalid_branch_or_commit_yaml_input/maven_invalid_branch.yaml similarity index 79% rename from tests/e2e/configurations/maven_invalid_branch.yaml rename to tests/integration/cases/invalid_branch_or_commit_yaml_input/maven_invalid_branch.yaml index 192a86251..0f183fb48 100644 --- a/tests/e2e/configurations/maven_invalid_branch.yaml +++ b/tests/integration/cases/invalid_branch_or_commit_yaml_input/maven_invalid_branch.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: diff --git a/tests/e2e/configurations/maven_invalid_digest.yaml b/tests/integration/cases/invalid_branch_or_commit_yaml_input/maven_invalid_commit.yaml similarity index 78% rename from tests/e2e/configurations/maven_invalid_digest.yaml rename to tests/integration/cases/invalid_branch_or_commit_yaml_input/maven_invalid_commit.yaml index be155f612..97da9ac4f 100644 --- a/tests/e2e/configurations/maven_invalid_digest.yaml +++ b/tests/integration/cases/invalid_branch_or_commit_yaml_input/maven_invalid_commit.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. target: diff --git a/tests/integration/cases/invalid_branch_or_commit_yaml_input/test.yaml b/tests/integration/cases/invalid_branch_or_commit_yaml_input/test.yaml new file mode 100644 index 000000000..871de42a8 --- /dev/null +++ b/tests/integration/cases/invalid_branch_or_commit_yaml_input/test.yaml @@ -0,0 +1,21 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Check the e2e status code of running with invalid branch or commit defined in the yaml configuration. + +steps: +- name: Run macaron analyze with yaml input containing invalid branch + kind: analyze + options: + command_args: + - -c + - maven_invalid_branch.yaml + expect_fail: true +- name: Run macaron analyze with yaml input containing invalid commit + kind: analyze + options: + command_args: + - -c + - maven_invalid_commit.yaml + expect_fail: true From 344f168c480bd750ad34a14d95fd71a1b7f8cdd4 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sun, 16 Jun 2024 19:04:57 +1000 Subject: [PATCH 45/47] chore: convert case apache_maven_yaml_input_no_deps_and_skip_deps Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 14 ------------- .../dependencies.json} | 0 .../maven_config.yaml | 0 .../test.yaml | 20 +++++++++++++++++++ 4 files changed, 20 insertions(+), 14 deletions(-) rename tests/{dependency_analyzer/expected_results/skipdep_apache_maven.json => integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/dependencies.json} (100%) rename tests/{dependency_analyzer/configurations => integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps}/maven_config.yaml (100%) create mode 100644 tests/integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index ed1de2db5..25728518c 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -120,20 +120,6 @@ $RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail # python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail -# Analyze apache/maven. -echo -e "\n==================================================================================" -echo "Run integration tests with configurations for apache/maven..." -echo -e "==================================================================================\n" - -echo -e "\n----------------------------------------------------------------------------------" -echo "apache/maven: Check the resolved dependency output when automatic dependency resolution is skipped." -echo -e "----------------------------------------------------------------------------------\n" -DEP_RESULT=$WORKSPACE/output/reports/maven/org_apache_maven/maven/dependencies.json -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/skipdep_apache_maven.json -run_macaron_clean $ANALYZE -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml --skip-deps || log_fail - -check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "Test using the default template file." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/dependency_analyzer/expected_results/skipdep_apache_maven.json b/tests/integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/dependencies.json similarity index 100% rename from tests/dependency_analyzer/expected_results/skipdep_apache_maven.json rename to tests/integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/dependencies.json diff --git a/tests/dependency_analyzer/configurations/maven_config.yaml b/tests/integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/maven_config.yaml similarity index 100% rename from tests/dependency_analyzer/configurations/maven_config.yaml rename to tests/integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/maven_config.yaml diff --git a/tests/integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/test.yaml b/tests/integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/test.yaml new file mode 100644 index 000000000..81ae769f7 --- /dev/null +++ b/tests/integration/cases/apache_maven_yaml_input_no_deps_and_skip_deps/test.yaml @@ -0,0 +1,20 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Check the e2e output JSON file with yaml input containing no dependency and skipping dependency resolution. + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -c + - maven_config.yaml + - --skip-deps +- name: Compare dependency report + kind: compare + options: + kind: deps_report + result: output/reports/maven/org_apache_maven/maven/dependencies.json + expected: dependencies.json From e8715d86b6a89e7b14ea60786bdf56edf22d2c57 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 22 Jun 2024 13:53:52 +1000 Subject: [PATCH 46/47] chore: convert case timyarkov_multibuild_test_maven Signed-off-by: Nathan Nguyen --- scripts/dev_scripts/integration_tests.sh | 13 -------- .../dev_scripts/integration_tests_docker.sh | 13 -------- .../dependencies.json} | 0 .../policy.dl} | 0 .../timyarkov_multibuild_test_maven/test.yaml | 33 +++++++++++++++++++ 5 files changed, 33 insertions(+), 26 deletions(-) rename tests/{dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_maven.json => integration/cases/timyarkov_multibuild_test_maven/dependencies.json} (100%) rename tests/{e2e/expected_results/maven/org.example/mock_maven_proj/1.0-SNAPSHOT/multibuild_test.dl => integration/cases/timyarkov_multibuild_test_maven/policy.dl} (100%) create mode 100644 tests/integration/cases/timyarkov_multibuild_test_maven/test.yaml diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 25728518c..5439190c9 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -86,19 +86,6 @@ echo -e "\n===================================================================== echo "Run integration tests without configurations" echo -e "==================================================================================\n" -echo -e "\n----------------------------------------------------------------------------------" -echo "timyarkov/multibuild_test: Analyzing Maven artifact with the repo path, the branch name and the commit digest" -echo "with dependency resolution using cyclonedx Maven plugins (defaults)." -echo -e "----------------------------------------------------------------------------------\n" -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_maven.json -DEP_RESULT=$WORKSPACE/output/reports/maven/org_example/mock_maven_proj/dependencies.json -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/maven/org.example/mock_maven_proj/1.0-SNAPSHOT/multibuild_test.dl - -run_macaron_clean $ANALYZE -purl pkg:maven/org.example/mock_maven_proj@1.0-SNAPSHOT?type=jar -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail -check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "apache/maven: Analyzing the repo path, the branch name and the commit digest with dependency resolution using cyclonedx maven plugin (default)." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 9c333b131..d9ee88ffd 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -38,19 +38,6 @@ echo "Run unit tests for the run_macaron.sh script" python $UNIT_TEST_SCRIPT || log_fail echo -e "\n----------------------------------------------------------------------------------" -echo -e "\n----------------------------------------------------------------------------------" -echo "timyarkov/multibuild_test: Analyzing Maven artifact with the repo path, the branch name and the commit digest" -echo "with dependency resolution using cyclonedx Maven plugins (defaults)." -echo -e "----------------------------------------------------------------------------------\n" -DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_maven.json -DEP_RESULT=$WORKSPACE/output/reports/maven/org_example/mock_maven_proj/dependencies.json -OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/maven/org.example/mock_maven_proj/1.0-SNAPSHOT/multibuild_test.dl -run_macaron_clean analyze -purl pkg:maven/org.example/mock_maven_proj@1.0-SNAPSHOT?type=jar -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail - -python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail - -$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail - echo -e "\n----------------------------------------------------------------------------------" echo "pkg:pypi/django@5.0.6: Analyzing the dependencies with virtual env provided as input." echo -e "----------------------------------------------------------------------------------\n" diff --git a/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_maven.json b/tests/integration/cases/timyarkov_multibuild_test_maven/dependencies.json similarity index 100% rename from tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test_maven.json rename to tests/integration/cases/timyarkov_multibuild_test_maven/dependencies.json diff --git a/tests/e2e/expected_results/maven/org.example/mock_maven_proj/1.0-SNAPSHOT/multibuild_test.dl b/tests/integration/cases/timyarkov_multibuild_test_maven/policy.dl similarity index 100% rename from tests/e2e/expected_results/maven/org.example/mock_maven_proj/1.0-SNAPSHOT/multibuild_test.dl rename to tests/integration/cases/timyarkov_multibuild_test_maven/policy.dl diff --git a/tests/integration/cases/timyarkov_multibuild_test_maven/test.yaml b/tests/integration/cases/timyarkov_multibuild_test_maven/test.yaml new file mode 100644 index 000000000..67a42255f --- /dev/null +++ b/tests/integration/cases/timyarkov_multibuild_test_maven/test.yaml @@ -0,0 +1,33 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Analyzing Maven artifact with the repo path, the branch name and the commit digest + with dependency resolution using cyclonedx Maven plugins (defaults). + +tags: +- docker + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/org.example/mock_maven_proj@1.0-SNAPSHOT?type=jar + - -rp + - https://github.com/timyarkov/multibuild_test + - -b + - main + - -d + - a8b0efe24298bc81f63217aaa84776c3d48976c5 +- name: Compare dependency resolution result + kind: compare + options: + kind: deps_report + result: output/reports/maven/org_example/mock_maven_proj/dependencies.json + expected: dependencies.json +- name: Run macaron verify-policy to verify passed/failed checks + kind: verify + options: + policy: policy.dl From d57e50cecc53a5ec970119d4be3ba3f469526e31 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Sat, 22 Jun 2024 15:16:32 +1000 Subject: [PATCH 47/47] chore: move the dependency installation of the utility script into Makefile Signed-off-by: Nathan Nguyen --- .github/workflows/_build_docker.yaml | 3 +-- Makefile | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_build_docker.yaml b/.github/workflows/_build_docker.yaml index 0ab977d48..d6a6b21cf 100644 --- a/.github/workflows/_build_docker.yaml +++ b/.github/workflows/_build_docker.yaml @@ -60,8 +60,7 @@ jobs: run: make build-docker - name: Install dependencies for integration test utility - run: | - python3 -m pip install ruamel.yaml cfgv + run: make setup-integration-test-utility-for-docker # Run the integration tests against the built Docker image. - name: Test the Docker image diff --git a/Makefile b/Makefile index 88663b62a..b47f97005 100644 --- a/Makefile +++ b/Makefile @@ -205,6 +205,12 @@ setup-github-actions: python -m pip install --upgrade wheel python -m pip install --upgrade --upgrade-strategy eager --editable .[actions] +# Install dependencies for the integration test utility script in workflow to +# test the docker image. +.PHONY: setup-integration-test-utility-for-docker +setup-integration-test-utility-for-docker: + python -m pip install ruamel.yaml cfgv + # Generate a Software Bill of Materials (SBOM). .PHONY: sbom sbom: requirements