[automation] Auto-update linters version, help and documentation #21101
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| ######################### | |
| ######################### | |
| ## Deploy Docker Image Linters ## | |
| ######################### | |
| ######################### | |
| # Documentation: | |
| # https://help.github.com/en/articles/workflow-syntax-for-github-actions | |
| # | |
| ####################################### | |
| # Start the job on all push to main # | |
| ####################################### | |
| name: "Build & Deploy - DEV linters" | |
| on: | |
| pull_request: | |
| ############### | |
| # Set the Job # | |
| ############### | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.ref_name }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| get-linters-matrix: | |
| name: Get Linters Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| jobs: ${{ steps.set-matrix.outputs.jobs }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| persist-credentials: false | |
| - name: Get PR title or commit message | |
| id: get-title | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_TITLE: ${{ github.event.pull_request.title }} | |
| PR_USER: ${{ github.event.pull_request.user.login }} | |
| HEAD_COMMIT_MSG: ${{ github.event.head_commit.message }} | |
| run: | | |
| if [ "$EVENT_NAME" = "pull_request" ]; then | |
| echo "title=${PR_TITLE}" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "title=${HEAD_COMMIT_MSG}" >> "${GITHUB_OUTPUT}" | |
| fi | |
| echo "pr_user=${PR_USER}" >> "${GITHUB_OUTPUT}" | |
| - name: Read linters_matrix.json and filter | |
| id: set-matrix | |
| run: | | |
| extract_pkg() { | |
| # Extract after 'upgrade', 'update dependency', 'update docker tag', etc. | |
| pkg=$(echo "$1" | sed -E 's/^chore\(deps\): (upgrade|update dependency|update docker tag|update .* plugin|update .* docker tag) ([^ ]+).*/\2/i') | |
| # If not found, try generic extraction for dependency lines | |
| if [ -z "$pkg" ]; then | |
| pkg=$(echo "$1" | sed -E 's/^chore\(deps\): update dependency ([^ ]+).*/\1/i') | |
| fi | |
| # Extract last segment after / or @ (e.g., friendsofphp/php-cs-fixer -> php-cs-fixer, @salesforce/plugin-packaging -> plugin-packaging) | |
| pkg=$(echo "$pkg" | sed -E 's/^.*[\/@]//') | |
| # Normalize: replace - and / with _, lowercase | |
| pkg=$(echo "$pkg" | sed 's/@//g; s/[-\/]/_/g' | tr '[:upper:]' '[:lower:]') | |
| echo "$pkg" | |
| } | |
| title="${STEPS_GET_TITLE_OUTPUTS_TITLE}" | |
| latest_commit_message=$(git log -1 --pretty=%B) | |
| echo "title=$title" | |
| echo "latest_commit_message=$latest_commit_message" | |
| matrix=$(jq -c '[.["linux/amd64"][]]' .automation/generated/linters_matrix.json) | |
| matrix_arm=$(jq -c '[.["linux/arm64"][]]' .automation/generated/linters_matrix.json) | |
| if echo "$latest_commit_message" | grep -q 'ARM'; then | |
| include_arm=true | |
| else | |
| include_arm=false | |
| fi | |
| pkg=$(extract_pkg "$title") | |
| echo "Extracted package: $pkg" | |
| # Detect dependabot / renovate PRs: bot author OR chore(deps) title prefix. | |
| is_bot_pr=false | |
| case "$PR_USER" in | |
| dependabot|"dependabot[bot]"|renovate|"renovate[bot]") | |
| is_bot_pr=true | |
| ;; | |
| esac | |
| if echo "$title" | grep -iq '^chore(deps)'; then | |
| is_bot_pr=true | |
| fi | |
| if [ -n "$pkg" ] && echo "$title" | grep -iq '^chore(deps)'; then | |
| matches=$(echo "$matrix" | jq -c --arg pkg "$pkg" '[.[] | select(tostring | match($pkg;"i"))]') | |
| if [ "$(echo "$matches" | jq 'length')" -gt 0 ]; then | |
| echo "Filtered linters: $matches" | |
| selected_linters="$matches" | |
| amd_jobs=$(jq -cn --argjson linters "${selected_linters}" '[ $linters[] | {linter: ., platform: "linux/amd64", runner: "ubuntu-latest"} ]') | |
| arm_jobs='[]' | |
| if [ "${include_arm}" = "true" ]; then | |
| arm_jobs=$(jq -cn --argjson linters "${selected_linters}" --argjson arm "${matrix_arm}" '[ $linters[] as $l | select($arm | index($l)) | {linter: $l, platform: "linux/arm64", runner: "ubuntu-24.04-arm"} ]') | |
| fi | |
| jobs=$(jq -cn --argjson amd "${amd_jobs}" --argjson arm "${arm_jobs}" '$amd + $arm') | |
| echo "jobs=$jobs" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| fi | |
| fi | |
| # Bot PR with no matching linter: skip linter jobs entirely. | |
| # deploy-DEV.yml runs the full test suite on merge, so re-running | |
| # every linter image on each dependabot/renovate PR is wasted CI. | |
| if [ "$is_bot_pr" = "true" ]; then | |
| echo "Bot PR with no matching linter, skipping per-linter jobs (deploy-DEV will run full tests)." | |
| echo "jobs=[]" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| fi | |
| # Human PR: build only the linter images impacted by the changed | |
| # files, when the change is clearly scoped (descriptor edit, generated | |
| # per-linter Dockerfile, test fixtures of a single linter). ANY file | |
| # outside the recognized patterns keeps the full matrix: when in | |
| # doubt, run everything rather than miss an issue. | |
| if [ -n "${PR_NUMBER}" ]; then | |
| changed_files=$(gh api --paginate "repos/${GH_REPO}/pulls/${PR_NUMBER}/files?per_page=100" --jq '.[].filename' 2>/dev/null || echo "__api_error__") | |
| if [ "$changed_files" = "__api_error__" ] || [ -z "$changed_files" ]; then | |
| echo "Could not list PR changed files, keeping the full matrix." | |
| else | |
| selected='[]' | |
| full_needed=false | |
| while IFS= read -r file; do | |
| [ -z "$file" ] && continue | |
| case "$file" in | |
| docs/*|*.md|mega-linter-runner/*|.automation/generated/*|TEMPLATES/*|.vscode/*|.devcontainer/*|.config/*|LICENSE|.gitignore|.gitpod*) | |
| # No impact on linter images | |
| ;; | |
| .github/workflows/deploy-DEV-linters.yml) | |
| echo "This workflow itself changed: full matrix" | |
| full_needed=true | |
| ;; | |
| .github/*) | |
| # Other workflows/config do not change linter images | |
| ;; | |
| megalinter/descriptors/schemas/*) | |
| echo "Descriptor schema changed ($file): full matrix" | |
| full_needed=true | |
| ;; | |
| megalinter/descriptors/*.megalinter-descriptor.yml) | |
| desc=$(basename "$file" .megalinter-descriptor.yml) | |
| matches=$(echo "$matrix" | jq -c --arg p "${desc}_" '[.[] | select(startswith($p))]') | |
| if [ "$(echo "$matches" | jq 'length')" -gt 0 ]; then | |
| selected=$(jq -cn --argjson a "$selected" --argjson b "$matches" '$a + $b') | |
| else | |
| echo "Descriptor $desc matches no linter in the matrix: full matrix" | |
| full_needed=true | |
| fi | |
| ;; | |
| linters/*/*) | |
| lname=$(echo "$file" | cut -d/ -f2) | |
| if echo "$matrix" | jq -e --arg l "$lname" 'index($l)' >/dev/null; then | |
| selected=$(jq -cn --argjson a "$selected" --arg l "$lname" '$a + [$l]') | |
| else | |
| echo "Unknown linter folder $lname: full matrix" | |
| full_needed=true | |
| fi | |
| ;; | |
| .automation/test/*) | |
| folder=$(echo "$file" | cut -d/ -f3) | |
| if echo "$matrix" | jq -e --arg l "$folder" 'index($l)' >/dev/null; then | |
| selected=$(jq -cn --argjson a "$selected" --arg l "$folder" '$a + [$l]') | |
| else | |
| # Test folder shared by several linters (e.g. shell): doubt -> full | |
| echo "Test folder $folder has no 1:1 matrix entry: full matrix" | |
| full_needed=true | |
| fi | |
| ;; | |
| *) | |
| # Core code, Dockerfile, flavors, build system, deps, ... | |
| echo "Core file changed ($file): full matrix" | |
| full_needed=true | |
| ;; | |
| esac | |
| done <<< "$changed_files" | |
| if [ "$full_needed" = "false" ]; then | |
| selected=$(echo "$selected" | jq -c 'unique') | |
| if [ "$(echo "$selected" | jq 'length')" -eq 0 ]; then | |
| echo "No linter image impacted by this PR, skipping per-linter jobs." | |
| echo "jobs=[]" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| fi | |
| echo "Path-filtered linters: $selected" | |
| amd_jobs=$(jq -cn --argjson linters "${selected}" '[ $linters[] | {linter: ., platform: "linux/amd64", runner: "ubuntu-latest"} ]') | |
| arm_jobs='[]' | |
| if [ "${include_arm}" = "true" ]; then | |
| arm_jobs=$(jq -cn --argjson linters "${selected}" --argjson arm "${matrix_arm}" '[ $linters[] as $l | select($arm | index($l)) | {linter: $l, platform: "linux/arm64", runner: "ubuntu-24.04-arm"} ]') | |
| fi | |
| jobs=$(jq -cn --argjson amd "${amd_jobs}" --argjson arm "${arm_jobs}" '$amd + $arm') | |
| echo "jobs=$jobs" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| fi | |
| fi | |
| fi | |
| # Default: return all | |
| echo "Change impacts shared code or file list unavailable, using all linters." | |
| amd_jobs=$(jq -cn --argjson linters "${matrix}" '[ $linters[] | {linter: ., platform: "linux/amd64", runner: "ubuntu-latest"} ]') | |
| arm_jobs='[]' | |
| if [ "${include_arm}" = "true" ]; then | |
| arm_jobs=$(jq -cn --argjson linters "${matrix_arm}" '[ $linters[] | {linter: ., platform: "linux/arm64", runner: "ubuntu-24.04-arm"} ]') | |
| fi | |
| jobs=$(jq -cn --argjson amd "${amd_jobs}" --argjson arm "${arm_jobs}" '$amd + $arm') | |
| echo "jobs=$jobs" >> "${GITHUB_OUTPUT}" | |
| env: | |
| STEPS_GET_TITLE_OUTPUTS_TITLE: ${{ steps.get-title.outputs.title }} | |
| PR_USER: ${{ steps.get-title.outputs.pr_user }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| GH_REPO: ${{ github.repository }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| build: | |
| # Name the Job | |
| name: DEV/Linters | |
| # Set the agent to run on | |
| runs-on: ${{ matrix.runner }} | |
| needs: get-linters-matrix | |
| permissions: | |
| contents: read | |
| # Require writing security events to upload SARIF file to security tab | |
| security-events: write | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 18 | |
| matrix: | |
| include: ${{ fromJson(needs.get-linters-matrix.outputs.jobs) }} | |
| # Only run this on the main repo | |
| if: | | |
| ( | |
| (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || | |
| (github.event_name == 'push' && github.repository == 'oxsecurity/megalinter') | |
| ) | |
| && !contains(github.event.head_commit.message, 'skip deploy') | |
| && !contains(github.event.head_commit.message, 'skip linters') | |
| ################## | |
| # Load all steps # | |
| ################## | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - name: Docker Metadata action | |
| uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 | |
| id: meta | |
| with: | |
| images: | | |
| ${{ github.repository }}-only-${{ matrix.linter }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 | |
| - name: Build Image | |
| id: build_linter | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 | |
| continue-on-error: true | |
| with: | |
| context: . | |
| file: linters/${{ matrix.linter }}/Dockerfile | |
| platforms: ${{ matrix.platform }} | |
| build-args: | | |
| BUILD_DATE=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} | |
| BUILD_VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} | |
| BUILD_REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} | |
| load: true | |
| push: false | |
| secrets: | | |
| GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| # Read-only: rely on BETA-linters' warm cache. PR builds don't | |
| # write their own cache — the resulting blobs would be read at | |
| # most once (by BETA-linters on merge) and burn space the warm | |
| # cache needs. | |
| cache-from: | | |
| type=gha,scope=beta-linter-${{ matrix.linter }}-${{ matrix.platform }},ignore-error=true | |
| # Fallback: rebuild from scratch when the warm cache returns a stale | |
| # manifest with evicted blobs ('blob sha256:...: not found'). | |
| - name: Build Image (no-cache fallback) | |
| if: ${{ steps.build_linter.outcome == 'failure' }} | |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 | |
| with: | |
| context: . | |
| file: linters/${{ matrix.linter }}/Dockerfile | |
| platforms: ${{ matrix.platform }} | |
| build-args: | | |
| BUILD_DATE=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} | |
| BUILD_VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} | |
| BUILD_REVISION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} | |
| load: true | |
| push: false | |
| secrets: | | |
| GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| ##################################### | |
| # Run Linter test cases # | |
| ##################################### | |
| - name: Run Test Cases | |
| shell: bash | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} | |
| GH_REPOSITORY: ${{ github.repository }} | |
| HEAD_REF: ${{ github.head_ref }} | |
| REF_NAME: ${{ github.ref_name }} | |
| MATRIX_LINTER: ${{ matrix.linter }} | |
| DOCKER_IMAGE: ${{ fromJson(steps.meta.outputs.json).tags[0] }} | |
| GH_SHA: ${{ github.sha }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| GITHUB_REPOSITORY=$([ "$EVENT_NAME" == "pull_request" ] && echo "$PR_HEAD_REPO" || echo "$GH_REPOSITORY") | |
| GITHUB_BRANCH=$([ "$EVENT_NAME" == "pull_request" ] && echo "$HEAD_REF" || echo "$REF_NAME") | |
| TEST_KEYWORDS_TO_USE_UPPER="${MATRIX_LINTER}" | |
| TEST_KEYWORDS_TO_USE="${TEST_KEYWORDS_TO_USE_UPPER,,}" | |
| docker image ls | |
| docker run -e TEST_CASE_RUN=true -e OUTPUT_FORMAT=text -e OUTPUT_FOLDER="${GH_SHA}" -e OUTPUT_DETAIL=detailed -e GITHUB_SHA="${GH_SHA}" -e GITHUB_REPOSITORY="${GITHUB_REPOSITORY}" -e GITHUB_BRANCH="${GITHUB_BRANCH}" -e GITHUB_TOKEN="${GITHUB_TOKEN}" -e TEST_KEYWORDS="${TEST_KEYWORDS_TO_USE}" -e MEGALINTER_VOLUME_ROOT="${GITHUB_WORKSPACE}" -v "/var/run/docker.sock:/var/run/docker.sock:rw" -v "${GITHUB_WORKSPACE}:/tmp/lint" "${DOCKER_IMAGE}" | |
| timeout-minutes: 30 | |
| ############################################## | |
| # Check Docker image security with Trivy # | |
| ############################################## | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 | |
| with: | |
| # renovate: datasource=github-releases depName=aquasecurity/trivy | |
| version: "v0.74.0" | |
| image-ref: "${{ fromJson(steps.meta.outputs.json).tags[0] }}" | |
| format: "table" | |
| exit-code: "1" | |
| ignore-unfixed: true | |
| scanners: vuln | |
| vuln-type: "os,library" | |
| severity: "CRITICAL,HIGH" | |
| timeout: 10m0s | |
| env: | |
| ACTIONS_RUNTIME_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # - name: Run OSV-Scanner vulnerability scanner | |
| # uses: google/osv-scanner-action/osv-scanner-action@v2.3.5 | |
| # with: | |
| # scan-args: |- | |
| # --docker ${{ fromJson(steps.meta.outputs.json).tags[0] }} |