Skip to content

fix(validate): catch typos in containers config and close self-scan UX gaps #356

fix(validate): catch typos in containers config and close self-scan UX gaps

fix(validate): catch typos in containers config and close self-scan UX gaps #356

Workflow file for this run

name: Test Composite Actions
# Comprehensive integration tests for all composite actions
# Uses matrix strategy to keep workflows consolidated while providing clear feedback
#
# DESIGN PRINCIPLES:
# - Single workflow, multiple matrix jobs for consolidation
# - Path-based triggers so only affected actions are tested on PR
# - Clear job naming so devs can quickly identify failures
# - Minimal test fixtures for fast feedback (<5 min per scanner)
# - Examples validated to ensure user-facing docs work
on:
pull_request:
paths:
- '.github/actions/**'
- '.github/workflows/**'
- 'tests/fixtures/**'
push:
branches: [main]
paths:
- '.github/actions/**'
- '.github/workflows/**'
workflow_dispatch:
inputs:
test_group:
description: 'Test group to run (all, sast, secrets, container, linters)'
required: false
default: 'all'
type: choice
options:
- all
- sast
- secrets
- container
- linters
- infrastructure
- dependencies
- supply-chain
permissions:
contents: read
security-events: write
actions: read
pull-requests: write
packages: read
# Cancel in-progress runs when a new commit is pushed
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
# Shared settings for all tests
PYTHON_VERSION: '3.12'
NODE_VERSION: '22'
# Disable actual security uploads during tests
ENABLE_CODE_SECURITY: 'false'
jobs:
# ============================================================================
# E2E Test Coverage Validation
# Reports which actions have E2E tests and which are missing
# Posts coverage report as PR comment and fails if coverage is incomplete
# ============================================================================
validate-e2e-coverage:
name: Validate E2E Coverage
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Check action E2E test coverage
id: coverage
run: |
# Build report content for both step summary and PR comment
report_file=$(mktemp)
{
echo "## E2E Test Coverage Report"
echo ""
} >> "$report_file"
# Get all actions (directories containing action.yml)
actions=$(find .github/actions -maxdepth 2 -name "action.yml" | while IFS= read -r f; do basename "$(dirname "$f")"; done | sort)
# Get tested actions from this workflow file
tested=$(grep -oP 'uses: \./\.github/actions/\K[^"]+' .github/workflows/test-actions.yml | sort -u)
# Actions that are tested indirectly or are utilities
# These are documented exceptions that don't need direct E2E tests
declare -A exceptions=(
["comment-pr"]="Utility action - tested indirectly by all scanner and linter actions"
["get-job-id"]="Utility action - tested indirectly by other jobs"
["scanner-container-summary"]="Tested as part of scanner-container"
["scanner-zap-summary"]="Tested as part of scanner-zap"
["security-summary"]="Tested in test-summary job"
["ai-summary"]="Manual-only action - triggered via dedicated ai-summary.yml workflow with user-supplied PR number"
)
{
echo "| Action | E2E Test Status | Notes |"
echo "|--------|-----------------|-------|"
} >> "$report_file"
missing=0
missing_list=""
total=0
covered=0
for action in $actions; do
total=$((total + 1))
if echo "$tested" | grep -q "^$action$"; then
echo "| \`$action\` | ✅ Tested | |" >> "$report_file"
covered=$((covered + 1))
elif [[ -v "exceptions[$action]" ]]; then
echo "| \`$action\` | ⚪ Exception | ${exceptions[$action]} |" >> "$report_file"
covered=$((covered + 1))
else
echo "| \`$action\` | ❌ Missing | Needs E2E test |" >> "$report_file"
missing=$((missing + 1))
missing_list="${missing_list}- \`${action}\`\n"
fi
done
coverage_pct=$((covered * 100 / total))
{
echo ""
echo "### Summary"
echo "- **Total Actions:** $total"
echo "- **Covered:** $covered"
echo "- **Missing E2E Tests:** $missing"
echo "- **Coverage:** ${coverage_pct}%"
} >> "$report_file"
if [ $missing -gt 0 ]; then
{
echo ""
echo "### ❌ Actions Missing E2E Tests"
echo -e "$missing_list"
echo ""
echo "> **Action Required:** Add E2E tests for the missing actions in \`test-actions.yml\` or add them to the exceptions list with justification."
} >> "$report_file"
else
{
echo ""
echo "### ✅ All Actions Have E2E Coverage"
} >> "$report_file"
fi
# Write to step summary
cat "$report_file" >> "$GITHUB_STEP_SUMMARY"
# Save report for PR comment step
cat "$report_file" >> "$GITHUB_WORKSPACE/coverage-report.md"
# Output for other steps/jobs
{
echo "missing=$missing"
echo "coverage=$coverage_pct"
echo "report_file=$GITHUB_WORKSPACE/coverage-report.md"
} >> "$GITHUB_OUTPUT"
- name: Post coverage report to PR
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Add comment header for identification (allows updating existing comment)
comment_marker="<!-- e2e-coverage-report -->"
# Check if comment already exists (use databaseId for REST API)
existing_comment=$(gh api "/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" --jq '.[] | select(.body | contains("<!-- e2e-coverage-report -->")) | .id' | head -1)
comment_body="${comment_marker}
$(cat "$GITHUB_WORKSPACE/coverage-report.md")"
if [ -n "$existing_comment" ]; then
# Update existing comment
gh api \
--method PATCH \
-H "Accept: application/vnd.github+json" \
"/repos/${{ github.repository }}/issues/comments/${existing_comment}" \
-f body="$comment_body"
echo "Updated existing PR comment"
else
# Create new comment
gh pr comment ${{ github.event.pull_request.number }} --body "$comment_body"
echo "Created new PR comment"
fi
- name: Fail if missing E2E tests
if: steps.coverage.outputs.missing != '0'
run: |
echo "::error::${STEPS_COVERAGE_OUTPUTS_MISSING} actions are missing E2E tests. Add tests or document exceptions."
exit 1
env:
STEPS_COVERAGE_OUTPUTS_MISSING: ${{ steps.coverage.outputs.missing }}
# ============================================================================
# SAST Scanners (Static Application Security Testing)
# Tests: scanner-bandit, scanner-opengrep
# ============================================================================
test-bandit:
name: SAST / bandit
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'sast') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run Bandit scanner
id: scan
uses: ./.github/actions/scanner-bandit
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
bandit_config_file: 'pyproject.toml'
- name: Verify scan completed
shell: bash
run: |
{
echo "## Bandit Test Results"
echo "✅ Scanner executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
test-opengrep:
name: SAST / opengrep
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'sast') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run Opengrep scanner
id: scan
uses: ./.github/actions/scanner-opengrep
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
- name: Verify scan completed
shell: bash
run: |
{
echo "## Opengrep Test Results"
echo "✅ Scanner executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
# CodeQL requires separate job due to unique setup
test-codeql:
name: SAST / codeql
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'sast') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 20
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run CodeQL scanner (Python)
id: scan
uses: ./.github/actions/scanner-codeql
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
language: 'python'
setup_python_version: ${{ env.PYTHON_VERSION }}
- name: Verify scan completed
shell: bash
run: |
{
echo "## CodeQL Test Results"
echo "✅ CodeQL scanner executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
# ============================================================================
# Secrets Detection
# Tests: scanner-gitleaks
# ============================================================================
test-secrets:
name: Secrets / gitleaks
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'secrets') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
fetch-depth: 0 # Full history for comprehensive secrets scanning
- name: Run Gitleaks scanner
id: scan
uses: ./.github/actions/scanner-gitleaks
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} # Optional
with:
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
gitleaks_config: '.gitleaks.toml'
- name: Verify scan completed
shell: bash
run: |
{
echo "## Gitleaks Test Results"
echo "✅ Gitleaks scanner executed successfully"
echo ""
} >> "$GITHUB_STEP_SUMMARY"
if [ -n "${STEPS_SCAN_OUTPUTS_SECRETS_COUNT}" ]; then
echo "**Secrets detected:** ${STEPS_SCAN_OUTPUTS_SECRETS_COUNT}" >> "$GITHUB_STEP_SUMMARY"
fi
env:
STEPS_SCAN_OUTPUTS_SECRETS_COUNT: ${{ steps.scan.outputs.secrets_count }}
# ============================================================================
# Infrastructure Scanners (IaC)
# Tests: scanner-trivy-iac, scanner-checkov
# ============================================================================
test-trivy-iac:
name: IaC / trivy-iac
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'infrastructure') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run Trivy IaC scanner
id: scan
uses: ./.github/actions/scanner-trivy-iac
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
- name: Verify scan completed
shell: bash
run: |
{
echo "## Trivy IaC Test Results"
echo "✅ Scanner executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
test-checkov:
name: IaC / checkov
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'infrastructure') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run Checkov scanner
id: scan
uses: ./.github/actions/scanner-checkov
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
iac_path: 'tests/fixtures/iac'
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
- name: Verify scan completed
shell: bash
run: |
{
echo "## Checkov Test Results"
echo "✅ Scanner executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
# ============================================================================
# Compliance
# Tests: scn-detector
# ============================================================================
test-scn-detector:
name: Compliance / scn-detector
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'infrastructure') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
fetch-depth: 0 # Required for SCN detector git diff analysis
- name: Create test IaC changes
shell: bash
run: |
mkdir -p terraform
cat > terraform/test.tf << 'EOF'
resource "compute_instance" "test" {
name = "test-instance"
size = "medium"
tags = {
Environment = "test"
Purpose = "e2e-testing"
}
}
EOF
git config user.name "GitHub Actions"
git config user.email "actions@github.com"
git add terraform/test.tf
git commit -m "test: Add test infrastructure"
- name: Run SCN detector
id: scan
uses: ./.github/actions/scn-detector
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
base_ref: 'HEAD~1'
head_ref: 'HEAD'
create_issues: false
post_pr_comment: false
enable_ai_fallback: false
fail_on_category: 'none'
- name: Verify scan completed
shell: bash
run: |
{
echo "## SCN Detector Test Results"
echo "✅ SCN detector executed successfully"
echo ""
echo "**Change Category:** \`${STEPS_SCAN_OUTPUTS_CHANGE_CATEGORY}\`"
echo "**Has Changes:** \`${STEPS_SCAN_OUTPUTS_HAS_CHANGES}\`"
} >> "$GITHUB_STEP_SUMMARY"
env:
STEPS_SCAN_OUTPUTS_CHANGE_CATEGORY: ${{ steps.scan.outputs.change_category }}
STEPS_SCAN_OUTPUTS_HAS_CHANGES: ${{ steps.scan.outputs.has_changes }}
# ============================================================================
# Container Scanners
# Tests: scanner-container with multiple tools
# ============================================================================
test-container:
name: Container / ${{ matrix.tool }}
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'container') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
tool:
- trivy
- grype
include:
- tool: trivy
image: ghcr.io/anchore/syft:latest
- tool: grype
image: ghcr.io/anchore/syft:latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run container scanner (${{ matrix.tool }})
id: scan
uses: ./.github/actions/scanner-container
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
image_ref: ${{ matrix.image }}
container_name: test-${{ matrix.tool }}
scanners: ${{ matrix.tool }}
registry_username: ${{ github.actor }}
registry_password: ${{ secrets.GITHUB_TOKEN }}
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
- name: Verify scan completed
shell: bash
run: |
{
echo "## Container Scanner Test (${{ matrix.tool }})"
echo "✅ Container scan completed successfully"
echo ""
echo "**Image:** \`${{ matrix.image }}\`"
} >> "$GITHUB_STEP_SUMMARY"
# ============================================================================
# SBOM Generation
# Tests: scanner-syft
# ============================================================================
test-syft:
name: SBOM / syft
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'container') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run Syft SBOM generator
id: scan
uses: ./.github/actions/scanner-syft
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
scan_image: 'alpine:latest'
output_format: 'cyclonedx-json'
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
- name: Verify SBOM generated
shell: bash
run: |
{
echo "## Syft SBOM Test Results"
echo "✅ SBOM generated successfully"
echo ""
echo "**Image:** \`alpine:latest\`"
echo "**Components found:** ${STEPS_SCAN_OUTPUTS_COMPONENT_COUNT}"
echo "**SBOM file:** ${STEPS_SCAN_OUTPUTS_SBOM_FILE}"
} >> "$GITHUB_STEP_SUMMARY"
env:
STEPS_SCAN_OUTPUTS_COMPONENT_COUNT: ${{ steps.scan.outputs.component_count }}
STEPS_SCAN_OUTPUTS_SBOM_FILE: ${{ steps.scan.outputs.sbom_file }}
# ============================================================================
# Linters
# Tests: All linter actions
# ============================================================================
test-linter-dockerfile:
name: Linter / dockerfile
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'linters') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run Dockerfile linter
uses: ./.github/actions/linter-dockerfile
continue-on-error: true
- name: Verify linter executed
shell: bash
run: |
{
echo "## Dockerfile Linter Test Results"
echo "✅ Linter executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
test-linter-python:
name: Linter / python
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'linters') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run Python linter
uses: ./.github/actions/linter-python
continue-on-error: true
- name: Verify linter executed
shell: bash
run: |
{
echo "## Python Linter Test Results"
echo "✅ Linter executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
test-linter-yaml:
name: Linter / yaml
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'linters') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run YAML linter
uses: ./.github/actions/linter-yaml
continue-on-error: true
- name: Verify linter executed
shell: bash
run: |
{
echo "## YAML Linter Test Results"
echo "✅ Linter executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
test-linter-javascript:
name: Linter / javascript
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'linters') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run JavaScript linter
uses: ./.github/actions/linter-javascript
continue-on-error: true
- name: Verify linter executed
shell: bash
run: |
{
echo "## JavaScript Linter Test Results"
echo "✅ Linter executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
test-linter-json:
name: Linter / json
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'linters') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run JSON linter
uses: ./.github/actions/linter-json
continue-on-error: true
- name: Verify linter executed
shell: bash
run: |
{
echo "## JSON Linter Test Results"
echo "✅ Linter executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
test-linter-terraform:
name: Linter / terraform
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'linters') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run Terraform linter
uses: ./.github/actions/linter-terraform
continue-on-error: true
- name: Verify linter executed
shell: bash
run: |
{
echo "## Terraform Linter Test Results"
echo "✅ Linter executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
# ============================================================================
# DAST Scanner (ZAP)
# Tests against a live app to validate end-to-end functionality
# ============================================================================
test-zap:
name: DAST / zap
runs-on: ubuntu-latest
if: >-
github.event_name == 'pull_request' ||
github.event_name == 'push' ||
inputs.test_group == 'all'
timeout-minutes: 25
services:
podinfo:
image: stefanprodan/podinfo:latest@sha256:187803cdf611a19d4fffbdf6a4260a01be4c09ffe9924b28902424fc0639ceb8
ports:
- 9898:9898
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Wait for podinfo to be ready
shell: bash
run: |
echo "Waiting for podinfo to be ready..."
for i in {1..30}; do
if curl -sf http://localhost:9898/healthz > /dev/null; then
echo "✅ Podinfo is ready"
exit 0
fi
echo "Attempt $i/30 - waiting..."
sleep 2
done
echo "❌ Podinfo failed to start"
exit 1
- name: Run ZAP scanner
id: scan
uses: ./.github/actions/scanner-zap
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
target_url: 'http://localhost:9898'
scan_type: 'baseline'
post_pr_comment: 'false'
fail_on_severity: 'none'
- name: Verify scan completed
shell: bash
run: |
{
echo "## ZAP DAST Test Results"
echo "✅ ZAP scanner executed successfully against podinfo"
echo ""
echo "**Target:** http://localhost:9898"
echo "**Scan type:** baseline"
} >> "$GITHUB_STEP_SUMMARY"
# ============================================================================
# Malware Scanner (ClamAV)
# Takes longer due to database updates - separate job
# ============================================================================
test-clamav:
name: Malware / clamav
runs-on: ubuntu-latest
if: >-
github.event_name == 'pull_request' ||
github.event_name == 'push' ||
inputs.test_group == 'all'
timeout-minutes: 20
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run ClamAV scanner
id: scan
uses: ./.github/actions/scanner-clamav
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
scan_path: 'tests/fixtures'
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
- name: Verify scan completed
shell: bash
run: |
{
echo "## ClamAV Test Results"
echo "✅ ClamAV scanner executed successfully"
echo ""
echo "**Files scanned:** ${STEPS_SCAN_OUTPUTS_SCANNED_COUNT}"
echo "**Scan status:** ${STEPS_SCAN_OUTPUTS_SCAN_STATUS}"
} >> "$GITHUB_STEP_SUMMARY"
env:
STEPS_SCAN_OUTPUTS_SCANNED_COUNT: ${{ steps.scan.outputs.scanned_count }}
STEPS_SCAN_OUTPUTS_SCAN_STATUS: ${{ steps.scan.outputs.scan_status }}
# ============================================================================
# Supply Chain Scanner
# Tests: scanner-supply-chain (zizmor + actionlint)
# ============================================================================
test-supply-chain:
name: Supply Chain / supply-chain
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'supply-chain') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run supply chain scanner
id: scan
uses: ./.github/actions/scanner-supply-chain
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
scan_path: '.'
persona: 'regular'
run_actionlint: 'true'
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
- name: Verify scan completed
shell: bash
run: |
{
echo "## Supply Chain Scanner Test Results"
echo "✅ Supply chain scanner executed successfully"
echo ""
echo "**High:** ${STEPS_SCAN_OUTPUTS_HIGH_COUNT}"
echo "**Medium:** ${STEPS_SCAN_OUTPUTS_MEDIUM_COUNT}"
echo "**Low:** ${STEPS_SCAN_OUTPUTS_LOW_COUNT}"
echo "**Info:** ${STEPS_SCAN_OUTPUTS_INFO_COUNT}"
echo "**Status:** ${STEPS_SCAN_OUTPUTS_SCAN_STATUS}"
} >> "$GITHUB_STEP_SUMMARY"
env:
STEPS_SCAN_OUTPUTS_HIGH_COUNT: ${{ steps.scan.outputs.high_count }}
STEPS_SCAN_OUTPUTS_MEDIUM_COUNT: ${{ steps.scan.outputs.medium_count }}
STEPS_SCAN_OUTPUTS_LOW_COUNT: ${{ steps.scan.outputs.low_count }}
STEPS_SCAN_OUTPUTS_INFO_COUNT: ${{ steps.scan.outputs.info_count }}
STEPS_SCAN_OUTPUTS_SCAN_STATUS: ${{ steps.scan.outputs.scan_status }}
# ============================================================================
# Dependency Scanners
# Tests: scanner-osv, scanner-dependency-review
# ============================================================================
test-osv:
name: Dependencies / osv
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'dependencies') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run OSV scanner
id: scan
uses: ./.github/actions/scanner-osv
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
scan_path: '.'
recursive: 'true'
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
# Exercises config_file input; filters out intentionally vulnerable test fixtures
config_file: 'tests/fixtures/osv-scanner-test.toml'
- name: Verify scan completed
shell: bash
run: |
{
echo "## OSV Scanner Test Results"
echo "✅ OSV scanner executed successfully"
echo ""
echo "**Critical:** ${STEPS_SCAN_OUTPUTS_CRITICAL_COUNT}"
echo "**High:** ${STEPS_SCAN_OUTPUTS_HIGH_COUNT}"
echo "**Medium:** ${STEPS_SCAN_OUTPUTS_MEDIUM_COUNT}"
echo "**Low:** ${STEPS_SCAN_OUTPUTS_LOW_COUNT}"
} >> "$GITHUB_STEP_SUMMARY"
env:
STEPS_SCAN_OUTPUTS_CRITICAL_COUNT: ${{ steps.scan.outputs.critical_count }}
STEPS_SCAN_OUTPUTS_HIGH_COUNT: ${{ steps.scan.outputs.high_count }}
STEPS_SCAN_OUTPUTS_MEDIUM_COUNT: ${{ steps.scan.outputs.medium_count }}
STEPS_SCAN_OUTPUTS_LOW_COUNT: ${{ steps.scan.outputs.low_count }}
test-dependency-review:
name: Dependencies / dependency-review
runs-on: ubuntu-latest
if: |
github.event_name == 'workflow_dispatch' &&
(inputs.test_group == 'all' || inputs.test_group == 'dependencies') ||
github.event_name != 'workflow_dispatch'
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run dependency-review scanner
id: scan
uses: ./.github/actions/scanner-dependency-review
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
post_pr_comment: 'false'
enable_code_security: ${{ env.ENABLE_CODE_SECURITY }}
fail_on_severity: 'none'
- name: Verify scan completed
shell: bash
run: |
# On non-PR events, dependency-review gracefully skips with zero counts
{
echo "## Dependency Review Test Results"
echo "✅ Dependency review scanner executed successfully"
echo ""
echo "**Event type:** ${{ github.event_name }}"
echo "**Critical:** ${STEPS_SCAN_OUTPUTS_CRITICAL_COUNT}"
echo "**High:** ${STEPS_SCAN_OUTPUTS_HIGH_COUNT}"
echo "**License violations:** ${STEPS_SCAN_OUTPUTS_LICENSE_VIOLATIONS}"
} >> "$GITHUB_STEP_SUMMARY"
env:
STEPS_SCAN_OUTPUTS_CRITICAL_COUNT: ${{ steps.scan.outputs.critical_count }}
STEPS_SCAN_OUTPUTS_HIGH_COUNT: ${{ steps.scan.outputs.high_count }}
STEPS_SCAN_OUTPUTS_LICENSE_VIOLATIONS: ${{ steps.scan.outputs.license_violations }}
# ============================================================================
# Summary Generators
# Tests: linting-summary
# Note: Other summary generators need upstream artifacts, tested separately
# ============================================================================
test-linting-summary:
name: Summary / linting-summary
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run linting-summary action
id: summary
uses: ./.github/actions/linting-summary
with:
post_pr_comment: 'false'
continue-on-error: true
- name: Verify summary action executed
shell: bash
run: |
{
echo "## Linting Summary Test Results"
echo "✅ Summary action executed"
} >> "$GITHUB_STEP_SUMMARY"
# ============================================================================
# Config Parsers
# Tests: parse-container-config, parse-zap-config
# ============================================================================
test-parse-container-config:
name: Parser / container-config
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run container-config parser
id: parse
uses: ./.github/actions/parse-container-config
with:
config_file: tests/fixtures/configs/container-config.yml
- name: Verify parser output
shell: bash
run: |
{
echo "## Container Config Parser Test Results"
echo "✅ Parser executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
test-parse-zap-config:
name: Parser / zap-config
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run zap-config parser
id: parse
uses: ./.github/actions/parse-zap-config
with:
config_file: tests/fixtures/configs/zap-config.yml
- name: Verify parser output
shell: bash
run: |
{
echo "## ZAP Config Parser Test Results"
echo "✅ Parser executed successfully"
} >> "$GITHUB_STEP_SUMMARY"
# ============================================================================
# Example Workflow Validation
# Validates that example workflows have correct syntax and references
# ============================================================================
validate-examples:
name: Validate Examples
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Set up Node.js
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
with:
node-version: ${{ env.NODE_VERSION }}
- name: Validate example workflow syntax
shell: bash
run: |
{
echo "## Example Workflow Validation"
echo ""
} >> "$GITHUB_STEP_SUMMARY"
ERRORS=0
for example in examples/*.yml; do
if [ -f "$example" ]; then
echo "Validating: $example"
# Check YAML syntax
if ! python -c "import yaml; yaml.safe_load(open('$example'))" 2>/dev/null; then
echo "❌ **$example** - Invalid YAML syntax" >> "$GITHUB_STEP_SUMMARY"
ERRORS=$((ERRORS + 1))
continue
fi
# Check for action references to this repo
if grep -q "huntridge-labs/argus" "$example"; then
# Verify the action paths exist
REFS=$(grep -oP '(?<=uses: huntridge-labs/argus)\.github/actions/[^@]+' "$example" || true)
for ref in $REFS; do
if [ ! -d "$ref" ]; then
echo "❌ **$example** - References non-existent action: $ref" >> "$GITHUB_STEP_SUMMARY"
ERRORS=$((ERRORS + 1))
fi
done
fi
echo "✅ $example" >> "$GITHUB_STEP_SUMMARY"
fi
done
echo "" >> "$GITHUB_STEP_SUMMARY"
if [ "$ERRORS" -eq 0 ]; then
echo "**All example workflows validated successfully!**" >> "$GITHUB_STEP_SUMMARY"
else
echo "**Found $ERRORS validation errors**" >> "$GITHUB_STEP_SUMMARY"
exit 1
fi
- name: Check for required inputs documentation
shell: bash
run: |
{
echo ""
echo "## Required Inputs Check"
echo ""
} >> "$GITHUB_STEP_SUMMARY"
# Check that each action's required inputs are documented in examples
for action_dir in .github/actions/scanner-* .github/actions/linter-*; do
if [ -d "$action_dir" ]; then
action_name=$(basename "$action_dir")
action_file="$action_dir/action.yml"
if [ -f "$action_file" ]; then
# Get required inputs
REQUIRED=$(python -c "
import yaml
with open('$action_file') as f:
data = yaml.safe_load(f)
inputs = data.get('inputs', {})
for name, config in inputs.items():
if config.get('required', False) and config.get('default') is None:
print(name)
" 2>/dev/null || true)
if [ -n "$REQUIRED" ]; then
echo "**$action_name** requires: $REQUIRED" >> "$GITHUB_STEP_SUMMARY"
fi
fi
fi
done
# ============================================================================
# Final Summary
# Aggregates results from all test jobs
# ============================================================================
test-summary:
name: Test Summary
runs-on: ubuntu-latest
needs:
- validate-e2e-coverage
- test-bandit
- test-opengrep
- test-codeql
- test-secrets
- test-trivy-iac
- test-checkov
- test-container
- test-syft
- test-linter-dockerfile
- test-linter-python
- test-linter-yaml
- test-linter-javascript
- test-linter-json
- test-linter-terraform
- test-zap
- test-clamav
- test-supply-chain
- test-osv
- test-dependency-review
- test-linting-summary
- test-parse-container-config
- test-parse-zap-config
- validate-examples
if: always()
steps:
- name: Generate Summary
shell: bash
env:
WORKFLOW_NAME: ${{ github.workflow }}
EVENT_NAME: ${{ github.event_name }}
REF_NAME: ${{ github.ref_name }}
NEEDS_VALIDATE_E2E_COVERAGE_RESULT: ${{ needs.validate-e2e-coverage.result }}
NEEDS_TEST_BANDIT_RESULT: ${{ needs.test-bandit.result }}
NEEDS_TEST_OPENGREP_RESULT: ${{ needs.test-opengrep.result }}
NEEDS_TEST_CODEQL_RESULT: ${{ needs.test-codeql.result }}
NEEDS_TEST_SECRETS_RESULT: ${{ needs.test-secrets.result }}
NEEDS_TEST_TRIVY_IAC_RESULT: ${{ needs.test-trivy-iac.result }}
NEEDS_TEST_CHECKOV_RESULT: ${{ needs.test-checkov.result }}
NEEDS_TEST_CONTAINER_RESULT: ${{ needs.test-container.result }}
NEEDS_TEST_SYFT_RESULT: ${{ needs.test-syft.result }}
NEEDS_TEST_LINTER_DOCKERFILE_RESULT: ${{ needs.test-linter-dockerfile.result }}
NEEDS_TEST_LINTER_PYTHON_RESULT: ${{ needs.test-linter-python.result }}
NEEDS_TEST_LINTER_YAML_RESULT: ${{ needs.test-linter-yaml.result }}
NEEDS_TEST_LINTER_JAVASCRIPT_RESULT: ${{ needs.test-linter-javascript.result }}
NEEDS_TEST_LINTER_JSON_RESULT: ${{ needs.test-linter-json.result }}
NEEDS_TEST_LINTER_TERRAFORM_RESULT: ${{ needs.test-linter-terraform.result }}
NEEDS_TEST_ZAP_RESULT: ${{ needs.test-zap.result }}
NEEDS_TEST_CLAMAV_RESULT: ${{ needs.test-clamav.result }}
NEEDS_TEST_SUPPLY_CHAIN_RESULT: ${{ needs.test-supply-chain.result }}
NEEDS_TEST_OSV_RESULT: ${{ needs.test-osv.result }}
NEEDS_TEST_DEPENDENCY_REVIEW_RESULT: ${{ needs.test-dependency-review.result }}
NEEDS_TEST_LINTING_SUMMARY_RESULT: ${{ needs.test-linting-summary.result }}
NEEDS_TEST_PARSE_CONTAINER_CONFIG_RESULT: ${{ needs.test-parse-container-config.result }}
NEEDS_TEST_PARSE_ZAP_CONFIG_RESULT: ${{ needs.test-parse-zap-config.result }}
NEEDS_VALIDATE_EXAMPLES_RESULT: ${{ needs.validate-examples.result }}
run: |
{
echo "# Composite Actions Test Summary"
echo ""
echo "| Category | Status |"
echo "|----------|--------|"
echo "| E2E Coverage Validation | ${{ needs.validate-e2e-coverage.result == 'success' && '✅' || '⚠️' }} ${NEEDS_VALIDATE_E2E_COVERAGE_RESULT} |"
echo "| Bandit (SAST) | ${{ needs.test-bandit.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_BANDIT_RESULT} |"
echo "| Opengrep (SAST) | ${{ needs.test-opengrep.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_OPENGREP_RESULT} |"
echo "| CodeQL (SAST) | ${{ needs.test-codeql.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_CODEQL_RESULT} |"
echo "| Gitleaks (Secrets) | ${{ needs.test-secrets.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_SECRETS_RESULT} |"
echo "| Trivy IaC | ${{ needs.test-trivy-iac.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_TRIVY_IAC_RESULT} |"
echo "| Checkov IaC | ${{ needs.test-checkov.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_CHECKOV_RESULT} |"
echo "| Container Scanners | ${{ needs.test-container.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_CONTAINER_RESULT} |"
echo "| Syft SBOM | ${{ needs.test-syft.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_SYFT_RESULT} |"
echo "| Dockerfile Linter | ${{ needs.test-linter-dockerfile.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_LINTER_DOCKERFILE_RESULT} |"
echo "| Python Linter | ${{ needs.test-linter-python.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_LINTER_PYTHON_RESULT} |"
echo "| YAML Linter | ${{ needs.test-linter-yaml.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_LINTER_YAML_RESULT} |"
echo "| JavaScript Linter | ${{ needs.test-linter-javascript.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_LINTER_JAVASCRIPT_RESULT} |"
echo "| JSON Linter | ${{ needs.test-linter-json.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_LINTER_JSON_RESULT} |"
echo "| Terraform Linter | ${{ needs.test-linter-terraform.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_LINTER_TERRAFORM_RESULT} |"
echo "| ZAP DAST | ${{ needs.test-zap.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_ZAP_RESULT} |"
echo "| ClamAV Malware | ${{ needs.test-clamav.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_CLAMAV_RESULT} |"
echo "| Supply Chain | ${{ needs.test-supply-chain.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_SUPPLY_CHAIN_RESULT} |"
echo "| OSV Dependencies | ${{ needs.test-osv.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_OSV_RESULT} |"
echo "| Dependency Review | ${{ needs.test-dependency-review.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_DEPENDENCY_REVIEW_RESULT} |"
echo "| Linting Summary | ${{ needs.test-linting-summary.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_LINTING_SUMMARY_RESULT} |"
echo "| Container Config Parser | ${{ needs.test-parse-container-config.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_PARSE_CONTAINER_CONFIG_RESULT} |"
echo "| ZAP Config Parser | ${{ needs.test-parse-zap-config.result == 'success' && '✅' || '⚠️' }} ${NEEDS_TEST_PARSE_ZAP_CONFIG_RESULT} |"
echo "| Example Validation | ${{ needs.validate-examples.result == 'success' && '✅' || '⚠️' }} ${NEEDS_VALIDATE_EXAMPLES_RESULT} |"
echo ""
echo "---"
echo ""
echo "**Workflow:** $WORKFLOW_NAME"
echo "**Trigger:** $EVENT_NAME"
echo "**Branch:** $REF_NAME"
} >> "$GITHUB_STEP_SUMMARY"
# ============================================================================
# Test Results Gate
# Single required status check for branch protection
# Fails if ANY test job failed
# ============================================================================
test-results:
name: Test Results
if: always()
needs:
- validate-e2e-coverage
- test-bandit
- test-opengrep
- test-codeql
- test-secrets
- test-trivy-iac
- test-checkov
- test-scn-detector
- test-container
- test-syft
- test-linter-dockerfile
- test-linter-python
- test-linter-yaml
- test-linter-javascript
- test-linter-json
- test-linter-terraform
- test-zap
- test-clamav
- test-supply-chain
- test-osv
- test-dependency-review
- test-linting-summary
- test-parse-container-config
- test-parse-zap-config
- validate-examples
- test-summary
runs-on: ubuntu-latest
steps:
- name: Check test results
run: |
if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
echo "::error::One or more test jobs failed"
exit 1
fi
echo "All test jobs passed"