Skip to content

Commit bf7dc90

Browse files
benmssbenmss
authored
chore: convert expectation files to policies and use in integration tests (#760)
Signed-off-by: Ben Selwyn-Smith <[email protected]>
1 parent c79098d commit bf7dc90

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+851
-14626
lines changed

‎scripts/dev_scripts/integration_tests.sh‎

Lines changed: 156 additions & 187 deletions
Large diffs are not rendered by default.

‎scripts/dev_scripts/integration_tests_docker.sh‎

Lines changed: 46 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,19 @@ RUN_MACARON_SCRIPT=$2
1414

1515
# The scripts to compare the results of the integration tests.
1616
COMPARE_DEPS=$WORKSPACE/tests/dependency_analyzer/compare_dependencies.py
17-
COMPARE_JSON_OUT=$WORKSPACE/tests/e2e/compare_e2e_result.py
1817
COMPARE_POLICIES=$WORKSPACE/tests/policy_engine/compare_policy_reports.py
1918
COMPARE_VSA=$WORKSPACE/tests/vsa/compare_vsa.py
2019
UNIT_TEST_SCRIPT=$WORKSPACE/scripts/dev_scripts/test_run_macaron_sh.py
20+
RUN_POLICY="$RUN_MACARON_SCRIPT verify-policy"
21+
DB=$WORKSPACE/output/macaron.db
2122

2223
RESULT_CODE=0
2324

25+
function run_macaron_clean() {
26+
rm $DB
27+
$RUN_MACARON_SCRIPT "$@"
28+
}
29+
2430
function log_fail() {
2531
printf "Error: FAILED integration test (line ${BASH_LINENO}) %s\n" $@
2632
RESULT_CODE=1
@@ -37,49 +43,47 @@ echo "with dependency resolution using cyclonedx Gradle plugin (default)."
3743
echo -e "----------------------------------------------------------------------------------\n"
3844
DEP_RESULT=$WORKSPACE/output/reports/github_com/timyarkov/multibuild_test/dependencies.json
3945
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test.json
40-
JSON_RESULT=$WORKSPACE/output/reports/github_com/timyarkov/multibuild_test/multibuild_test.json
41-
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/multibuild_test/multibuild_test.json
42-
$RUN_MACARON_SCRIPT analyze -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail
46+
OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/multibuild_test/multibuild_test.dl
47+
run_macaron_clean analyze -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail
4348

4449
python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
4550

46-
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
51+
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
4752

4853
echo -e "\n----------------------------------------------------------------------------------"
4954
echo "apache/maven: Check the resolved dependency output with config for cyclonedx maven plugin (default)."
5055
echo -e "----------------------------------------------------------------------------------\n"
5156
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
5257
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json
5358

54-
$RUN_MACARON_SCRIPT analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || log_fail
59+
run_macaron_clean analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || log_fail
5560
python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
5661

5762
echo -e "\n----------------------------------------------------------------------------------"
5863
echo "apache/maven: e2e using the local repo path, the branch name and the commit digest without dependency resolution."
5964
echo -e "----------------------------------------------------------------------------------\n"
60-
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
61-
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
65+
OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/maven/maven.dl
66+
67+
run_macaron_clean -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
6268

63-
$RUN_MACARON_SCRIPT -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
64-
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
69+
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
6570

6671
echo -e "\n----------------------------------------------------------------------------------"
6772
echo "apache/maven: Check the e2e output JSON file with config and no dependency analyzing."
6873
echo -e "----------------------------------------------------------------------------------\n"
69-
JSON_RESULT_DIR=$WORKSPACE/output/reports/github_com/apache/maven
70-
JSON_EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/maven
74+
EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/maven
7175

7276
declare -a COMPARE_FILES=(
73-
"maven.json"
74-
"guava.json"
75-
"mockito.json"
77+
"maven.dl"
78+
"guava.dl"
79+
"mockito.dl"
7680
)
7781

78-
$RUN_MACARON_SCRIPT analyze -c $WORKSPACE/tests/e2e/configurations/maven_config.yaml --skip-deps || log_fail
82+
run_macaron_clean analyze -c $WORKSPACE/tests/e2e/configurations/maven_config.yaml --skip-deps || log_fail
7983

8084
for i in "${COMPARE_FILES[@]}"
8185
do
82-
python $COMPARE_JSON_OUT $JSON_RESULT_DIR/$i $JSON_EXPECT_DIR/$i || log_fail
86+
$RUN_POLICY -d $DB -f $EXPECT_DIR/$i || log_fail
8387
done
8488

8589
echo -e "\n----------------------------------------------------------------------------------"
@@ -89,63 +93,47 @@ SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_
8993
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json
9094
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
9195

92-
$RUN_MACARON_SCRIPT analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom $SBOM_FILE || log_fail
96+
run_macaron_clean analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom $SBOM_FILE || log_fail
9397

9498
python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
9599

96100
echo -e "\n----------------------------------------------------------------------------------"
97101
echo "apache/maven: Analyzing with PURL and repository path without dependency resolution."
98102
echo -e "----------------------------------------------------------------------------------\n"
99-
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.json
100-
JSON_RESULT=$WORKSPACE/output/reports/maven/apache/maven/maven.json
101-
$RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
103+
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.dl
104+
run_macaron_clean analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
102105

103-
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
106+
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
104107

105108
echo -e "\n----------------------------------------------------------------------------------"
106109
echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped."
107110
echo "The CUE expectation file is provided as a single file path."
108111
echo -e "----------------------------------------------------------------------------------\n"
109-
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.json
110-
JSON_RESULT=$WORKSPACE/output/reports/github_com/urllib3/urllib3/urllib3.json
112+
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl
111113
EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/urllib3_PASS.cue
112-
$RUN_MACARON_SCRIPT analyze -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
114+
run_macaron_clean analyze -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
113115

114-
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
116+
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
115117

116118
echo -e "\n----------------------------------------------------------------------------------"
117119
echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped."
118120
echo "The CUE expectation file should be found via the directory path."
119121
echo -e "----------------------------------------------------------------------------------\n"
120-
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.json
121-
JSON_RESULT=$WORKSPACE/output/reports/github_com/urllib3/urllib3/urllib3.json
122+
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl
122123
EXPECTATION_DIR=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/
123-
$RUN_MACARON_SCRIPT analyze -pe $EXPECTATION_DIR -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
124+
run_macaron_clean analyze -pe $EXPECTATION_DIR -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
124125

125-
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
126+
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
126127

127128
echo -e "\n----------------------------------------------------------------------------------"
128129
echo "Test verifying CUE provenance expectation for ossf/scorecard"
129130
echo -e "----------------------------------------------------------------------------------\n"
130-
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.json
131-
JSON_RESULT=$WORKSPACE/output/reports/github/ossf/scorecard/scorecard.json
131+
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.dl
132132
DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/scorecard.ini
133133
EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue
134-
$RUN_MACARON_SCRIPT -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/[email protected] --skip-deps || log_fail
134+
run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/[email protected] --skip-deps || log_fail
135135

136-
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
137-
138-
echo -e "\n----------------------------------------------------------------------------------"
139-
echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped"
140-
echo "and CUE file is provided as expectation."
141-
echo -e "----------------------------------------------------------------------------------\n"
142-
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json
143-
JSON_RESULT=$WORKSPACE/output/reports/github_com/slsa-framework/slsa-verifier/slsa-verifier.json
144-
EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue
145-
DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini
146-
$RUN_MACARON_SCRIPT -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
147-
148-
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
136+
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
149137

150138
echo -e "\n----------------------------------------------------------------------------------"
151139
echo "Run policy CLI with scorecard results."
@@ -160,11 +148,21 @@ $RUN_MACARON_SCRIPT verify-policy -f $POLICY_FILE -d "$WORKSPACE/output/macaron.
160148
python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail
161149
python "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail
162150

151+
echo -e "\n----------------------------------------------------------------------------------"
152+
echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped"
153+
echo "and CUE file is provided as expectation."
154+
echo -e "----------------------------------------------------------------------------------\n"
155+
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.dl
156+
EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue
157+
DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini
158+
run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
159+
160+
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail
161+
163162
echo -e "\n----------------------------------------------------------------------------------"
164163
echo "behnazh-w/example-maven-app as a local and remote repository"
165164
echo "Test the Witness and GitHub provenances as an input, Cue expectation validation, Policy CLI and VSA generation."
166165
echo -e "----------------------------------------------------------------------------------\n"
167-
RUN_POLICY="macaron verify-policy"
168166
POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/example-maven-project/policy.dl
169167
POLICY_RESULT=$WORKSPACE/output/policy_report.json
170168
POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/example-maven-project/example_maven_project_policy_report.json
@@ -179,7 +177,7 @@ WITNESS_PROVENANCE_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/resources/vali
179177
git clone https://github.com/behnazh-w/example-maven-app.git $WORKSPACE/output/git_repos/local_repos/example-maven-app || log_fail
180178

181179
# Check the Witness provenance.
182-
$RUN_MACARON_SCRIPT analyze -pf $WITNESS_PROVENANCE_FILE -pe $WITNESS_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/[email protected]?type=jar --repo-path example-maven-app --skip-deps || log_fail
180+
run_macaron_clean analyze -pf $WITNESS_PROVENANCE_FILE -pe $WITNESS_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/[email protected]?type=jar --repo-path example-maven-app --skip-deps || log_fail
183181

184182
# Test the remote repo with GitHub provenance.
185183
GITHUB_EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/github-example-maven-project.cue

‎src/macaron/policy_engine/prelude/helper_rules.dl‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. */
1+
/* Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. */
22
/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */
33

44
/* Souffle datalog rules to assist in policy authoring */
@@ -9,6 +9,9 @@
99
.decl check_passed(component_id: number, check_name: symbol)
1010
check_passed(component_id, check_name) :- check_result(_, check_name, 1, component_id).
1111

12+
.decl check_failed(component_id: number, check_name: symbol)
13+
check_failed(component_id, check_name) :- check_result(_, check_name, 0, component_id).
14+
1215
/**
1316
* Check name is valid.
1417
*/
@@ -42,6 +45,12 @@ transitive_dependency(parent, dependency) :-
4245
.decl is_repo(repo_id: number, repo_complete_name: symbol, component_id: number)
4346
is_repo(repo_id, repo_complete_name, component_id) :- repository(repo_id, component_id, repo_complete_name, _, _, _, _, _, _, _, _, _, _).
4447

48+
/**
49+
* Extract the id and URL from the repository relation.
50+
*/
51+
.decl is_repo_url(repo_id: number, url: symbol)
52+
is_repo_url(repo_id, url) :- repository(_, repo_id, _, _, _, _, _, url, _, _, _, _, _).
53+
4554

4655
/**
4756
* A convenience relation to find analyses conducted on a specific repository.

‎tests/e2e/expected_results/docker_test/docker_test.dl‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */
2+
/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */
3+
4+
#include "prelude.dl"
5+
6+
Policy("test_policy", component_id, "") :-
7+
check_passed(component_id, "mcn_build_as_code_1"),
8+
check_passed(component_id, "mcn_build_script_1"),
9+
check_passed(component_id, "mcn_build_service_1"),
10+
check_passed(component_id, "mcn_version_control_system_1"),
11+
check_failed(component_id, "mcn_infer_artifact_pipeline_1"),
12+
check_failed(component_id, "mcn_provenance_available_1"),
13+
check_failed(component_id, "mcn_provenance_derived_commit_1"),
14+
check_failed(component_id, "mcn_provenance_derived_repo_1"),
15+
check_failed(component_id, "mcn_provenance_expectation_1"),
16+
check_failed(component_id, "mcn_provenance_level_three_1"),
17+
check_failed(component_id, "mcn_provenance_witness_level_one_1"),
18+
check_failed(component_id, "mcn_trusted_builder_level_three_1"),
19+
is_repo_url(component_id, "https://github.com/timyarkov/docker_test").
20+
21+
apply_policy_to("test_policy", component_id) :-
22+
is_component(component_id, "pkg:github.com/timyarkov/docker_test@404a51a2f38c4470af6b32e4e00b5318c2d7c0cc").

0 commit comments

Comments
 (0)