Skip to content

Commit

Permalink
test: add integration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
tromai committed Feb 11, 2024
1 parent 0500c88 commit e0cc4b2
Show file tree
Hide file tree
Showing 4 changed files with 261 additions and 0 deletions.
23 changes: 23 additions & 0 deletions scripts/dev_scripts/integration_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,18 @@ do
check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT_DIR/$i $JSON_EXPECT_DIR/$i || log_fail
done

echo -e "\n----------------------------------------------------------------------------------"
echo "micronaut-projects/micronaut-test: Analyzing the repo path when automatic dependency resolution is skipped"
echo "and all provenance checks are excluded."
echo -e "----------------------------------------------------------------------------------\n"
JSON_RESULT=$WORKSPACE/output/reports/github_com/micronaut-projects/micronaut-test/micronaut-test.json
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/micronaut-test/micronaut-test_provenance_checks_excluded.json
DEFAULTS_FILE=$WORKSPACE/tests/e2e/configurations/exclude_provenance_checks.ini

$RUN_MACARON -dp $DEFAULTS_FILE analyze -rp https://github.com/micronaut-projects/micronaut-test/ -d 7679d10b4073a3b842b6c56877c35fa8cd10acff --skip-deps || log_fail

check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

# TODO: uncomment the test below after resolving https://github.com/oracle/macaron/issues/60.
# echo -e "\n----------------------------------------------------------------------------------"
# echo "micronaut-projects/micronaut-test: Check the resolved dependency output with config for cyclonedx gradle plugin (default)."
Expand Down Expand Up @@ -577,6 +589,17 @@ fi
rm -rf "$SOURCE_REPO"
rm -rf "$TARGET_REPO"

echo -e "\n----------------------------------------------------------------------------------"
echo "Running the analysis with all checks excluded should returns an error code."
echo -e "----------------------------------------------------------------------------------\n"
$RUN_MACARON -dp tests/e2e/configurations/exclude_all_checks.ini analyze -rp https://github.com/apache/maven --skip-deps

if [ $? -eq 0 ];
then
echo -e "Expect non-zero status code but got $?."
log_fail
fi

echo -e "\n----------------------------------------------------------------------------------"
echo "apache/maven: test analyzing without the environment variable GITHUB_TOKEN being set."
echo -e "----------------------------------------------------------------------------------\n"
Expand Down
6 changes: 6 additions & 0 deletions tests/e2e/configurations/exclude_all_checks.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

[analysis.checks]
exclude = *
include = *
6 changes: 6 additions & 0 deletions tests/e2e/configurations/exclude_provenance_checks.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

[analysis.checks]
exclude = mcn_provenance_available_1
include = *
Original file line number Diff line number Diff line change
@@ -0,0 +1,226 @@
{
"metadata": {
"timestamps": "2024-02-11 15:51:21",
"has_passing_check": true,
"run_checks": [
"mcn_infer_artifact_pipeline_1",
"mcn_build_as_code_1",
"mcn_version_control_system_1",
"mcn_build_service_1",
"mcn_build_script_1",
"mcn_trusted_builder_level_three_1"
],
"check_tree": {
"mcn_version_control_system_1": {
"mcn_trusted_builder_level_three_1": {
"mcn_build_as_code_1": {
"mcn_infer_artifact_pipeline_1": {},
"mcn_build_service_1": {
"mcn_build_script_1": {}
}
}
}
},
"mcn_provenance_available_1": {
"mcn_provenance_witness_level_one_1": {},
"mcn_provenance_level_three_1": {},
"mcn_provenance_expectation_1": {}
}
}
},
"target": {
"info": {
"full_name": "pkg:github.com/micronaut-projects/micronaut-test@7679d10b4073a3b842b6c56877c35fa8cd10acff",
"local_cloned_path": "git_repos/github_com/micronaut-projects/micronaut-test",
"remote_path": "https://github.com/micronaut-projects/micronaut-test",
"branch": null,
"commit_hash": "7679d10b4073a3b842b6c56877c35fa8cd10acff",
"commit_date": "2023-11-07T06:43:31+01:00"
},
"provenances": {
"is_inferred": true,
"content": {
"github_actions": [
{
"_type": "https://in-toto.io/Statement/v0.1",
"subject": [],
"predicateType": "https://slsa.dev/provenance/v0.2",
"predicate": {
"builder": {
"id": "https://github.com/micronaut-projects/micronaut-test/blob/7679d10b4073a3b842b6c56877c35fa8cd10acff/.github/workflows/gradle.yml"
},
"buildType": "Custom github_actions",
"invocation": {
"configSource": {
"uri": "https://github.com/micronaut-projects/micronaut-test@refs/heads/None",
"digest": {
"sha1": "7679d10b4073a3b842b6c56877c35fa8cd10acff"
},
"entryPoint": "https://github.com/micronaut-projects/micronaut-test/blob/7679d10b4073a3b842b6c56877c35fa8cd10acff/.github/workflows/gradle.yml"
},
"parameters": {},
"environment": {}
},
"buildConfig": {
"jobID": "build",
"stepID": "\ud83d\udce6 Publish to Sonatype Snapshots"
},
"metadata": {
"buildInvocationId": "",
"buildStartedOn": "<TIMESTAMP>",
"buildFinishedOn": "<TIMESTAMP>",
"completeness": {
"parameters": "false",
"environment": "false",
"materials": "false"
},
"reproducible": "false"
},
"materials": [
{
"uri": "<URI>",
"digest": {}
}
]
}
}
],
"Maven Central Registry": []
}
},
"checks": {
"summary": {
"DISABLED": 0,
"FAILED": 2,
"PASSED": 4,
"SKIPPED": 0,
"UNKNOWN": 0
},
"results": [
{
"check_id": "mcn_build_as_code_1",
"check_description": "The build definition and configuration executed by the build service is verifiably derived from text file definitions stored in a version control system.",
"slsa_requirements": [
"Build as code - SLSA Level 3"
],
"justification": [
{
"The target repository uses build tool gradle to deploy": "https://github.com/micronaut-projects/micronaut-test/blob/7679d10b4073a3b842b6c56877c35fa8cd10acff/.github/workflows/gradle.yml",
"The build is triggered by": "https://github.com/micronaut-projects/micronaut-test/blob/7679d10b4073a3b842b6c56877c35fa8cd10acff/.github/workflows/gradle.yml"
},
"Deploy command: ['./gradlew', 'publishToSonatype', 'docs', '--no-daemon']",
"However, could not find a passing workflow run."
],
"result_type": "PASSED"
},
{
"check_id": "mcn_build_script_1",
"check_description": "Check if the target repo has a valid build script.",
"slsa_requirements": [
"Scripted Build - SLSA Level 1"
],
"justification": [
"Check mcn_build_script_1 is set to PASSED because mcn_build_service_1 PASSED."
],
"result_type": "PASSED"
},
{
"check_id": "mcn_build_service_1",
"check_description": "Check if the target repo has a valid build service.",
"slsa_requirements": [
"Build service - SLSA Level 2"
],
"justification": [
"Check mcn_build_service_1 is set to PASSED because mcn_build_as_code_1 PASSED."
],
"result_type": "PASSED"
},
{
"check_id": "mcn_version_control_system_1",
"check_description": "Check whether the target repo uses a version control system.",
"slsa_requirements": [
"Version controlled - SLSA Level 2"
],
"justification": [
{
"This is a Git repository": "https://github.com/micronaut-projects/micronaut-test"
}
],
"result_type": "PASSED"
},
{
"check_id": "mcn_infer_artifact_pipeline_1",
"check_description": "Detects potential pipelines from which an artifact is published.",
"slsa_requirements": [
"Build as code - SLSA Level 3"
],
"justification": [
"Unable to find a publishing timestamp for the artifact."
],
"result_type": "FAILED"
},
{
"check_id": "mcn_trusted_builder_level_three_1",
"check_description": "Check whether the target uses a trusted SLSA level 3 builder.",
"slsa_requirements": [
"Hermetic - SLSA Level 4",
"Isolated - SLSA Level 3",
"Parameterless - SLSA Level 4",
"Ephemeral environment - SLSA Level 3"
],
"justification": [
"Could not find a trusted level 3 builder as a GitHub Actions workflow."
],
"result_type": "FAILED"
}
]
}
},
"dependencies": {
"analyzed_deps": 0,
"unique_dep_repos": 0,
"checks_summary": [
{
"check_id": "mcn_version_control_system_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_provenance_witness_level_one_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_build_as_code_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_infer_artifact_pipeline_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_trusted_builder_level_three_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_provenance_level_three_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_build_script_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_provenance_expectation_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_provenance_available_1",
"num_deps_pass": 0
},
{
"check_id": "mcn_build_service_1",
"num_deps_pass": 0
}
],
"dep_status": []
}
}

0 comments on commit e0cc4b2

Please sign in to comment.