Skip to content

Commit 19f6679

Browse files
tromaitromai
committed
test: add integration tests for exclude/include
Signed-off-by: Trong Nhan Mai <[email protected]>
1 parent 2cc75c0 commit 19f6679

File tree

4 files changed

+233
-0
lines changed

4 files changed

+233
-0
lines changed

‎scripts/dev_scripts/integration_tests.sh

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,29 @@ then
418418
log_fail
419419
fi
420420

421+
echo -e "\n----------------------------------------------------------------------------------"
422+
echo "Test running the analysis with all checks excluded."
423+
echo -e "----------------------------------------------------------------------------------\n"
424+
$RUN_MACARON -dp tests/e2e/configurations/exclude_all_checks.ini analyze -rp https://github.com/apache/maven --skip-deps
425+
426+
if [ $? -eq 0 ];
427+
then
428+
echo -e "Expect non-zero status code but got $?."
429+
log_fail
430+
fi
431+
432+
echo -e "\n----------------------------------------------------------------------------------"
433+
echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped"
434+
echo "and provenance checks are excluded."
435+
echo -e "----------------------------------------------------------------------------------\n"
436+
JSON_RESULT=$WORKSPACE/output/reports/github_com/slsa-framework/slsa-verifier/slsa-verifier.json
437+
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_provenance_checks_excluded.json
438+
DEFAULTS_FILE=$WORKSPACE/tests/e2e/configurations/exclude_provenance_checks.ini
439+
440+
$RUN_MACARON -dp $DEFAULTS_FILE analyze -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail
441+
442+
$COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
443+
421444
# Testing the custom YAML-based provenance expectation verifier.
422445
echo -e "\n----------------------------------------------------------------------------------"
423446
echo "Test verifying YAML-based provenance expectation."

‎tests/e2e/configurations/exclude_all_checks.ini

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
3+
4+
[analysis.checks]
5+
exclude = *
6+
include = *

‎tests/e2e/configurations/exclude_provenance_checks.ini

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
3+
4+
[analysis.checks]
5+
exclude = mcn_provenance_available_1
6+
include = *

‎tests/e2e/expected_results/slsa-verifier/slsa-verifier_provenance_checks_excluded.json

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
{
2+
"metadata": {
3+
"timestamps": "2023-06-14 14:48:26",
4+
"check_tree": {
5+
"mcn_version_control_system_1": {
6+
"mcn_trusted_builder_level_three_1": {
7+
"mcn_build_as_code_1": {
8+
"mcn_build_service_1": {
9+
"mcn_build_script_1": {}
10+
}
11+
}
12+
}
13+
},
14+
"mcn_provenance_available_1": {
15+
"mcn_provenance_level_three_1": {
16+
"mcn_provenance_expectation_1": {}
17+
}
18+
}
19+
},
20+
"excluded_checks": [
21+
"mcn_provenance_level_three_1",
22+
"mcn_provenance_expectation_1",
23+
"mcn_provenance_available_1"
24+
]
25+
},
26+
"target": {
27+
"info": {
28+
"full_name": "slsa-framework/slsa-verifier",
29+
"local_cloned_path": "../../../../output/git_repos/github_com/slsa-framework/slsa-verifier",
30+
"remote_path": "https://github.com/slsa-framework/slsa-verifier",
31+
"branch": "main",
32+
"commit_hash": "fc50b662fcfeeeb0e97243554b47d9b20b14efac",
33+
"commit_date": "2022-10-04T01:00:02+00:00"
34+
},
35+
"provenances": {
36+
"is_inferred": true,
37+
"content": {
38+
"github_actions": [
39+
{
40+
"_type": "https://in-toto.io/Statement/v0.1",
41+
"subject": [],
42+
"predicateType": "https://slsa.dev/provenance/v0.2",
43+
"predicate": {
44+
"builder": {
45+
"id": "slsa-framework/slsa-github-generator/.github/workflows/[email protected]"
46+
},
47+
"buildType": "Trusted github_actions",
48+
"invocation": {
49+
"configSource": {
50+
"uri": "https://github.com/slsa-framework/slsa-verifier@refs/heads/main",
51+
"digest": {
52+
"sha1": "fc50b662fcfeeeb0e97243554b47d9b20b14efac"
53+
},
54+
"entryPoint": "https://github.com/slsa-framework/slsa-verifier/blob/fc50b662fcfeeeb0e97243554b47d9b20b14efac/.github/workflows/release.yml"
55+
},
56+
"parameters": {},
57+
"environment": {}
58+
},
59+
"buildConfig": {},
60+
"metadata": {
61+
"buildInvocationId": "",
62+
"buildStartedOn": "<TIMESTAMP>",
63+
"buildFinishedOn": "<TIMESTAMP>",
64+
"completeness": {
65+
"parameters": "false",
66+
"environment": "false",
67+
"materials": "false"
68+
},
69+
"reproducible": "false"
70+
},
71+
"materials": [
72+
{
73+
"uri": "<URI>",
74+
"digest": {}
75+
}
76+
]
77+
}
78+
}
79+
]
80+
}
81+
},
82+
"checks": {
83+
"summary": {
84+
"DISABLED": 0,
85+
"FAILED": 0,
86+
"PASSED": 5,
87+
"SKIPPED": 0,
88+
"UNKNOWN": 0
89+
},
90+
"results": [
91+
{
92+
"check_id": "mcn_build_as_code_1",
93+
"check_description": "The build definition and configuration executed by the build service is verifiably derived from text file definitions stored in a version control system.",
94+
"slsa_requirements": [
95+
"Build as code - SLSA Level 3"
96+
],
97+
"justification": [
98+
"Check mcn_build_as_code_1 is set to PASSED because mcn_trusted_builder_level_three_1 PASSED."
99+
],
100+
"result_type": "PASSED"
101+
},
102+
{
103+
"check_id": "mcn_build_script_1",
104+
"check_description": "Check if the target repo has a valid build script.",
105+
"slsa_requirements": [
106+
"Scripted Build - SLSA Level 1"
107+
],
108+
"justification": [
109+
"Check mcn_build_script_1 is set to PASSED because mcn_build_service_1 PASSED."
110+
],
111+
"result_type": "PASSED"
112+
},
113+
{
114+
"check_id": "mcn_build_service_1",
115+
"check_description": "Check if the target repo has a valid build service.",
116+
"slsa_requirements": [
117+
"Build service - SLSA Level 2"
118+
],
119+
"justification": [
120+
"Check mcn_build_service_1 is set to PASSED because mcn_build_as_code_1 PASSED."
121+
],
122+
"result_type": "PASSED"
123+
},
124+
{
125+
"check_id": "mcn_trusted_builder_level_three_1",
126+
"check_description": "Check whether the target uses a trusted SLSA level 3 builder.",
127+
"slsa_requirements": [
128+
"Hermetic - SLSA Level 4",
129+
"Isolated - SLSA Level 3",
130+
"Parameterless - SLSA Level 4",
131+
"Ephemeral environment - SLSA Level 3"
132+
],
133+
"justification": [
134+
{
135+
"Found trusted builder GitHub Actions: slsa-framework/slsa-github-generator/.github/workflows/[email protected] triggered by": "https://github.com/slsa-framework/slsa-verifier/blob/fc50b662fcfeeeb0e97243554b47d9b20b14efac/.github/workflows/release.yml"
136+
},
137+
"However, could not find a passing workflow run."
138+
],
139+
"result_type": "PASSED"
140+
},
141+
{
142+
"check_id": "mcn_version_control_system_1",
143+
"check_description": "Check whether the target repo uses a version control system.",
144+
"slsa_requirements": [
145+
"Version controlled - SLSA Level 2"
146+
],
147+
"justification": [
148+
{
149+
"This is a Git repository": "https://github.com/slsa-framework/slsa-verifier"
150+
}
151+
],
152+
"result_type": "PASSED"
153+
}
154+
]
155+
}
156+
},
157+
"dependencies": {
158+
"analyzed_deps": 0,
159+
"unique_dep_repos": 0,
160+
"checks_summary": [
161+
{
162+
"check_id": "mcn_version_control_system_1",
163+
"num_deps_pass": 0
164+
},
165+
{
166+
"check_id": "mcn_build_as_code_1",
167+
"num_deps_pass": 0
168+
},
169+
{
170+
"check_id": "mcn_trusted_builder_level_three_1",
171+
"num_deps_pass": 0
172+
},
173+
{
174+
"check_id": "mcn_provenance_level_three_1",
175+
"num_deps_pass": 0
176+
},
177+
{
178+
"check_id": "mcn_build_script_1",
179+
"num_deps_pass": 0
180+
},
181+
{
182+
"check_id": "mcn_provenance_expectation_1",
183+
"num_deps_pass": 0
184+
},
185+
{
186+
"check_id": "mcn_provenance_available_1",
187+
"num_deps_pass": 0
188+
},
189+
{
190+
"check_id": "mcn_build_service_1",
191+
"num_deps_pass": 0
192+
}
193+
],
194+
"dep_status": []
195+
},
196+
"policies_passed": [],
197+
"policies_failed": []
198+
}

0 commit comments

Comments
 (0)