build.yml

name: Build

# Add package name and optional publish for manual builds
run-name: >-
  ${{ github.event_name == 'workflow_dispatch' &&
      format(
        '{0} ({1})',
        (inputs.publish && 'Build and Publish' || 'Build'),
        inputs.package
      )
      || '' }}


on:
  workflow_dispatch:
    inputs:
      publish:
        description: 'Publish to repository'
        required: false
        default: false
        type: boolean
      package:
        description: 'Package to build'
        required: true
        default: 'adminer'
      add_dsm73_builds:
        description: 'Include DSM 7.3 archs'
        required: false
        default: false
        type: boolean
      add_dsm72_builds:
        description: 'Include DSM 7.2 archs'
        required: false
        default: false
        type: boolean
      add_dsm71_builds:
        description: 'Include DSM 7.1 archs'
        required: false
        default: true
        type: boolean
      add_dsm70_builds:
        description: 'Include DSM 7.0 archs'
        required: false
        default: false
        type: boolean
      add_dsm62_builds:
        description: 'Include DSM 6.2 archs'
        required: false
        default: true
        type: boolean
      add_dsm61_builds:
        description: 'Include DSM 6.1 archs'
        required: false
        default: false
        type: boolean
      add_dsm52_builds:
        description: 'Include DSM 5.2 archs'
        required: false
        default: false
        type: boolean
      add_srm13_builds:
        description: 'Include SRM 1.3 archs'
        required: false
        default: false
        type: boolean
      add_srm12_builds:
        description: 'Include SRM 1.2 archs'
        required: false
        default: false
        type: boolean
      add_noarch_builds:
        description: 'Include noarch packages'
        required: false
        default: true
        type: boolean
      add_noarch_dsm73_builds:
        description: 'Include noarch DSM 7.3 packages'
        required: false
        default: false
        type: boolean
      add_noarch_dsm72_builds:
        description: 'Include noarch DSM 7.2 packages'
        required: false
        default: false
        type: boolean
  pull_request:
    paths:
      - 'spk/**'
      - 'cross/**'
      - 'python/**'
      - 'native/**'
  push:
    branches:
      - "**"
    paths:
      - 'spk/**'
      - 'cross/**'
      - 'python/**'
      - 'native/**'

env:
  # Per-arch build-time budget (seconds): 20700 = 5h45m, below GitHub's 6h job
  # kill. build.sh hard-stops the in-progress build at this ceiling so completed
  # .spk/diagnostics still upload and the remaining packages are reported.
  BUILD_BUDGET_SECONDS: "20700"

# Cancel a previous, now-superseded run only while a pull request is still a
# Draft, so iterative pushes during development don't pile up. Every other run
# (non-draft PR, master/push, manual workflow_dispatch) gets a unique per-run
# group: a shared concurrency group always constrains — at most one running
# plus one pending run, with older pending runs evicted — even when
# cancel-in-progress is false, which would serialize non-draft PR builds,
# silently drop queued master builds and forbid parallel manual builds.
concurrency:
  group: >-
    ${{ github.event_name == 'pull_request' && github.event.pull_request.draft
        && format('build-{0}-{1}', github.workflow, github.ref)
        || format('build-run-{0}', github.run_id) }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }}

jobs:
  # Decide whether the build should run at all. Upstream events (pull_request,
  # or push to SynoCommunity/spksrc) and manual workflow_dispatch always build.
  # A fork's own push build is skipped while the branch has an open *draft* PR
  # upstream: during development the authoritative full build still runs on the
  # upstream PR, so re-running the fork's narrow build only wastes runner time.
  # A push event carries no PR context, hence the API lookup here.
  gate:
    name: Gate
    runs-on: ubuntu-latest
    outputs:
      should_build: ${{ steps.check.outputs.should_build }}
    steps:
      - id: check
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [ "${{ github.repository }}" = "SynoCommunity/spksrc" ] || \
             [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            echo "Upstream or manual run -> build."
            echo "should_build=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # Fork push: look up the branch's open PR in the upstream repo and skip
          # the build while that PR is a draft. Any lookup failure falls back to
          # building (fail-open).
          draft=$(gh api \
            "/repos/SynoCommunity/spksrc/pulls?head=${{ github.repository_owner }}:${{ github.ref_name }}&state=open" \
            --jq '[.[].draft] | first // false' 2>/dev/null || echo false)
          if [ "$draft" = "true" ]; then
            echo "Fork push for a draft PR (${{ github.ref_name }}) -> skip build."
            echo "should_build=false" >> "$GITHUB_OUTPUT"
          else
            echo "Fork push, PR not draft (or none) -> build."
            echo "should_build=true" >> "$GITHUB_OUTPUT"
          fi

  prepare:
    name: Prepare for Build
    runs-on: ubuntu-latest
    needs: gate
    if: needs.gate.outputs.should_build == 'true'
    outputs:
      arch_packages: ${{ steps.dependencies.outputs.arch_packages }}
      noarch_packages: ${{ steps.dependencies.outputs.noarch_packages }}
      arch_min_dsm72_packages: ${{ steps.dependencies.outputs.arch_min_dsm72_packages }}
      noarch_min_dsm72_packages: ${{ steps.dependencies.outputs.noarch_min_dsm72_packages }}
      arch_min_dsm73_packages: ${{ steps.dependencies.outputs.arch_min_dsm73_packages }}
      noarch_min_dsm73_packages: ${{ steps.dependencies.outputs.noarch_min_dsm73_packages }}
      has_arch_packages: ${{ steps.dependencies.outputs.has_arch_packages }}
      has_noarch_packages: ${{ steps.dependencies.outputs.has_noarch_packages }}
      has_arch_min_dsm72_packages: ${{ steps.dependencies.outputs.has_arch_min_dsm72_packages }}
      has_noarch_min_dsm72_packages: ${{ steps.dependencies.outputs.has_noarch_min_dsm72_packages }}
      has_arch_min_dsm73_packages: ${{ steps.dependencies.outputs.has_arch_min_dsm73_packages }}
      has_noarch_min_dsm73_packages: ${{ steps.dependencies.outputs.has_noarch_min_dsm73_packages }}
      distrib_hash: ${{ steps.distrib-hash.outputs.hash }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Get changed spk_packages and dependency_folders for pull request
        if: github.event_name == 'pull_request'
        id: getchanges_pr
        run: |
          git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | grep -oP "^spk/\K[^\/]*" | sort -u | xargs
          echo "spk_packages=$(git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | grep -oP "^spk/\K[^\/]*" | sort -u | xargs)" >> $GITHUB_OUTPUT
          git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | grep -oP "^(cross|python|native)/[^\/]*" | sort -u | xargs
          echo "dependency_folders=$(git diff --no-commit-id --name-only -r origin/master...${{github.event.pull_request.head.sha}} | grep -oP "^(cross|python|native)/[^\/]*" | sort -u | xargs)" >> $GITHUB_OUTPUT

      - name: Get changed spk_packages and dependency_folders of last commit for push
        if: github.event_name == 'push'
        id: getchanges_push
        run: |
          git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | grep -oP "^spk/\K[^\/]*" | sort -u | xargs
          echo "spk_packages=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | grep -oP "^spk/\K[^\/]*" | sort -u | xargs)" >> $GITHUB_OUTPUT
          git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | grep -oP "^(cross|python|native)/[^\/]*" | sort -u | xargs
          echo "dependency_folders=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | grep -oP "^(cross|python|native)/[^\/]*" | sort -u | xargs)" >> $GITHUB_OUTPUT

      - name: Evaluate dependencies
        id: dependencies
        run: ./.github/actions/prepare.sh
        env:
          GH_SPK_PACKAGES: ${{ steps.getchanges_push.outputs.spk_packages }} ${{ steps.getchanges_pr.outputs.spk_packages }}
          GH_DEPENDENCY_FOLDERS: ${{ steps.getchanges_push.outputs.dependency_folders }} ${{ steps.getchanges_pr.outputs.dependency_folders }}
          USER_SPK_TO_BUILD: ${{ github.event.inputs.package }}

      - name: Restore distrib cache
        id: cache-restore
        uses: actions/cache/restore@v6
        with:
          path: distrib
          # Use a placeholder key that will never match for the exact key,
          # forcing GitHub to use restore-keys to find the best available cache
          key: distrib-placeholder-${{ github.run_id }}
          restore-keys: |
            distrib-

      - name: Download source files
        run: ./.github/actions/download.sh
        env:
          DOWNLOAD_PACKAGES: ${{ steps.dependencies.outputs.download_packages }}
          ARCH_PACKAGES: ${{ steps.dependencies.outputs.arch_packages }}
          NOARCH_PACKAGES: ${{ steps.dependencies.outputs.noarch_packages }}

      - name: Compute distrib hash
        id: distrib-hash
        run: |
          # Compute a hash based on file paths and sizes (not content) for performance.
          # This is fast even with thousands of files since we don't read file contents.
          # Including file size ensures that if a corrupted file (.wrong) is replaced
          # by a fresh download with different size, the hash will change.
          # Format: "path size" per line, sorted for reproducibility, then hashed.
          if [ -d "distrib" ] && [ -n "$(ls -A distrib 2>/dev/null)" ]; then
            HASH=$(find distrib -type f -printf '%p %s\n' | sort | sha256sum | cut -d' ' -f1)
          else
            HASH="empty"
          fi
          echo "hash=$HASH" >> $GITHUB_OUTPUT
          echo "Distrib cache hash: $HASH"

      - name: Check if cache already exists
        id: cache-check
        run: |
          # Check if a cache with this exact hash already exists to avoid duplication.
          # We search for the exact key pattern to see if this content was already cached.
          CACHE_KEY="distrib-${{ steps.distrib-hash.outputs.hash }}"
          if gh cache list --json key | jq -e --arg key "$CACHE_KEY" '.[] | select(.key == $key)' > /dev/null 2>&1; then
            echo "Cache with key $CACHE_KEY already exists, skipping save"
            echo "save_cache=false" >> $GITHUB_OUTPUT
          else
            echo "New cache content detected, will save with key: $CACHE_KEY"
            echo "save_cache=true" >> $GITHUB_OUTPUT
          fi
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Save distrib cache
        if: steps.cache-check.outputs.save_cache == 'true'
        uses: actions/cache/save@v6
        with:
          path: distrib
          key: distrib-${{ steps.distrib-hash.outputs.hash }}

  set-defaults:
    name: Set Defaults
    runs-on: ubuntu-latest
    needs: prepare
    outputs:
      matrix: ${{ steps.defaults.outputs.matrix }}
      has_entries: ${{ steps.defaults.outputs.has_entries }}
    steps:
      - id: defaults
        run: |
          # Retrieve inputs and package availability
          add_dsm73_builds=${{ github.event.inputs.add_dsm73_builds || 'false' }}
          add_dsm72_builds=${{ github.event.inputs.add_dsm72_builds || 'false' }}
          add_dsm71_builds=${{ github.event.inputs.add_dsm71_builds || 'false' }}
          add_dsm70_builds=${{ github.event.inputs.add_dsm70_builds || 'false' }}
          add_dsm62_builds=${{ github.event.inputs.add_dsm62_builds || 'false' }}
          add_dsm61_builds=${{ github.event.inputs.add_dsm61_builds || 'false' }}
          add_dsm52_builds=${{ github.event.inputs.add_dsm52_builds || 'false' }}
          add_srm13_builds=${{ github.event.inputs.add_srm13_builds || 'false' }}
          add_srm12_builds=${{ github.event.inputs.add_srm12_builds || 'false' }}
          add_noarch_builds=${{ github.event.inputs.add_noarch_builds || 'false' }}
          add_noarch_dsm72_builds=${{ github.event.inputs.add_noarch_dsm72_builds || 'false' }}
          add_noarch_dsm73_builds=${{ github.event.inputs.add_noarch_dsm73_builds || 'false' }}
          has_arch_packages=${{ needs.prepare.outputs.has_arch_packages || 'false' }}
          has_arch_min_dsm72_packages=${{ needs.prepare.outputs.has_arch_min_dsm72_packages || 'false' }}
          has_arch_min_dsm73_packages=${{ needs.prepare.outputs.has_arch_min_dsm73_packages || 'false' }}
          has_noarch_packages=${{ needs.prepare.outputs.has_noarch_packages || 'false' }}
          has_noarch_min_dsm72_packages=${{ needs.prepare.outputs.has_noarch_min_dsm72_packages || 'false' }}
          has_noarch_min_dsm73_packages=${{ needs.prepare.outputs.has_noarch_min_dsm73_packages || 'false' }}

          # Dynamic logic for automatic builds
          if [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
            # Enable DSM 7.3 builds if arch packages require minimum DSM 7.3
            add_dsm73_builds=$([ "$has_arch_min_dsm73_packages" == "true" ] && echo "true" || echo "false")

            # Enable DSM 7.2 builds if arch packages require minimum DSM 7.2
            add_dsm72_builds=$([ "$has_arch_min_dsm72_packages" == "true" ] && echo "true" || echo "false")

            # Enable DSM 7.1 builds if arch packages exist
            add_dsm71_builds=$([ "$has_arch_packages" == "true" ] && echo "true" || echo "false")

            # Enable DSM 6.2 builds if arch packages exist
            add_dsm62_builds=$([ "$has_arch_packages" == "true" ] && echo "true" || echo "false")

            # Enable noarch builds if noarch packages exist
            add_noarch_builds=$([ "$has_noarch_packages" == "true" ] && echo "true" || echo "false")

            # Enable noarch DSM 7.3 builds if noarch packages require minimum DSM 7.3
            add_noarch_dsm73_builds=$([ "$has_noarch_min_dsm73_packages" == "true" ] && echo "true" || echo "false")

            # Enable noarch DSM 7.2 builds if noarch packages require minimum DSM 7.2
            add_noarch_dsm72_builds=$([ "$has_noarch_min_dsm72_packages" == "true" ] && echo "true" || echo "false")
          fi

          # Build matrix
          matrix=$(jq -n '{"include": []}')

          # Helper function to add entries to the matrix
          add_to_matrix() {
            matrix=$(echo "$matrix" | jq --arg arch "$1" '.include += [{"arch": $arch}]')
          }

          # Add noarch builds
          if [ "$add_noarch_builds" == "true" ]; then
            add_to_matrix "noarch-1.1"
            add_to_matrix "noarch-3.1"
            add_to_matrix "noarch-6.1"
            add_to_matrix "noarch-7.0"
          fi

          # Add noarch DSM 7.2 builds
          if [ "$add_noarch_dsm72_builds" == "true" ]; then
            add_to_matrix "noarch-7.2"
          fi

          # Add noarch DSM 7.3 builds
          if [ "$add_noarch_dsm73_builds" == "true" ]; then
            add_to_matrix "noarch-7.3"
          fi

          # Add DSM 7.3 builds
          if [ "$add_dsm73_builds" == "true" ]; then
            add_to_matrix "x64-7.3"
            add_to_matrix "aarch64-7.3"
            add_to_matrix "armv7-7.3"
          fi

          # Add DSM 7.2 builds
          if [ "$add_dsm72_builds" == "true" ]; then
            add_to_matrix "x64-7.2"
            add_to_matrix "aarch64-7.2"
            add_to_matrix "armv7-7.2"
          fi

          # Add DSM 7.1 builds
          if [ "$add_dsm71_builds" == "true" ]; then
            add_to_matrix "x64-7.1"
            add_to_matrix "aarch64-7.1"
            add_to_matrix "evansport-7.1"
            add_to_matrix "armv7-7.1"
            add_to_matrix "comcerto2k-7.1"
          fi

          # Add DSM 7.0 builds
          if [ "$add_dsm70_builds" == "true" ]; then
            add_to_matrix "x64-7.0"
            add_to_matrix "aarch64-7.0"
            add_to_matrix "evansport-7.0"
            add_to_matrix "armv7-7.0"
            add_to_matrix "comcerto2k-7.0"
          fi

          # Add DSM 6.2 builds
          if [ "$add_dsm62_builds" == "true" ]; then
            add_to_matrix "x64-6.2.4"
            add_to_matrix "aarch64-6.2.4"
            add_to_matrix "evansport-6.2.4"
            add_to_matrix "armv7-6.2.4"
            add_to_matrix "hi3535-6.2.4"
            add_to_matrix "88f6281-6.2.4"
            add_to_matrix "qoriq-6.2.4"
          fi

          # Add DSM 6.1 builds
          if [ "$add_dsm61_builds" == "true" ]; then
            add_to_matrix "x64-6.1"
            add_to_matrix "aarch64-6.1"
            add_to_matrix "evansport-6.1"
            add_to_matrix "armv7-6.1"
            add_to_matrix "hi3535-6.1"
            add_to_matrix "88f6281-6.1"
            add_to_matrix "qoriq-6.1"
          fi

          # Add DSM 5.2 builds
          if [ "$add_dsm52_builds" == "true" ]; then
            add_to_matrix "x86-5.2"
            add_to_matrix "88f6281-5.2"
            add_to_matrix "ppc853x-5.2"
          fi

          # Add SRM 1.3 builds
          if [ "$add_srm13_builds" == "true" ]; then
            add_to_matrix "aarch64-1.3"
            add_to_matrix "armv7-1.3"
          fi

          # Add SRM 1.2 builds
          if [ "$add_srm12_builds" == "true" ]; then
            add_to_matrix "armv7-1.2"
          fi

          # Output the final matrix
          echo "matrix=$(echo $matrix | jq -c)" >> $GITHUB_OUTPUT

          # Output if the matrix has entries
          echo "has_entries=$(jq -e '.include | length > 0' <<< "$matrix" > /dev/null && echo true || echo false)" >> $GITHUB_OUTPUT

  build:
    name: Build
    needs: [prepare, set-defaults]
    runs-on: ubuntu-latest
    if: needs.set-defaults.outputs.has_entries == 'true'
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.set-defaults.outputs.matrix) }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      # Free up disk space on GitHub runner before build starts
      - name: Free Disk Space (Ubuntu)
        run: |
          # Capture initial disk space before cleanup
          {
            echo "=========================================="
            echo "DISK SPACE BEFORE CLEANUP"
            echo "=========================================="
            df -h
            echo ""
            echo "=== Largest directories in workspace ==="
            du -h -d 2 /home/runner/work 2>/dev/null | sort -hr | head -20 || true
          } > disk_usage_start.txt
          
          # Also display to console
          cat disk_usage_start.txt
          
          echo ""
          echo "=== Removing unnecessary packages and files ==="
          
          # Remove Android SDK/NDK (takes ~10-15GB)
          sudo rm -rf /usr/local/lib/android || true
          
          # Remove .NET runtime (takes ~2GB)
          sudo rm -rf /usr/share/dotnet || true
          
          # Remove Haskell (takes ~5GB)
          sudo rm -rf /opt/ghc || true
          sudo rm -rf /usr/local/.ghcup || true
          
          # Remove large tool caches
          sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
          
          # Remove Docker images we don't need (keep space for our build image)
          docker rmi $(docker image ls -aq) 2>/dev/null || true
          
          # Remove swap file if exists (can free 4GB+)
          sudo swapoff -a || true
          sudo rm -f /swapfile /mnt/swapfile || true
          
          # Clean apt cache
          sudo apt-get clean || true
          
          # Remove large packages we don't need
          sudo apt-get remove -y '^aspnetcore-.*' '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' 'azure-cli' 'google-cloud-sdk' 'google-chrome-stable' 'firefox' 'powershell' 'mono-devel' 'temurin-.*' || true
          sudo apt-get autoremove -y || true
          sudo apt-get clean || true
          
          # Capture disk space after cleanup
          {
            echo ""
            echo "=========================================="
            echo "DISK SPACE AFTER CLEANUP"
            echo "=========================================="
            df -h
            echo ""
            echo "=== Space freed calculation ==="
            df -h / | tail -1 | awk '{print "Available space: " $4}'
          } >> disk_usage_start.txt
          
          # Display cleanup results to console
          echo ""
          echo "=== Disk space after cleanup ==="
          df -h
          echo ""
          echo "=== Space freed calculation ==="
          df -h / | tail -1 | awk '{print "Available space: " $4}'

      - name: Cache toolchains
        uses: actions/cache@v6
        # Do not cache qoriq toolchain. It must be built every time to install custom rust toolchain
        if: ${{ contains(matrix.arch,'qoriq') == false }}
        with:
          path: toolchain/*/work
          key: toolchain-${{ matrix.arch }}-v4-${{ hashFiles(format('toolchain/syno-{0}/digests',matrix.arch)) }}

      # Check disk space after toolchain cache restore
      - name: Disk Space After Cache Restore
        run: |
          echo "=== Disk Space after Cache ==="
          df -h

      - name: Restore distrib cache
        uses: actions/cache/restore@v6
        with:
          path: distrib
          # Use a placeholder key that will never match for the exact key,
          # forcing GitHub to use restore-keys to find the best available cache
          key: distrib-placeholder-${{ github.run_id }}
          restore-keys: |
            distrib-

      # Check disk space after distrib cache restore
      - name: Disk Space After Distrib Restore
        run: |
          echo "=== Disk Space after Distrib ==="
          df -h
          du -sh distrib 2>/dev/null || echo "No distrib directory"

      - name: Build Package (based on changed files)
        # We don't want to stop the build on errors.
        # Errors are reported in "Build Status"
        continue-on-error: true
        # Hard backstop below the 6h job kill in case build.sh's own budget can't
        # stop in time (e.g. a single package hangs); the always() steps below
        # still run and upload whatever completed.
        timeout-minutes: 350
        uses: docker://ghcr.io/synocommunity/spksrc:latest
        with:
          entrypoint: ./.github/actions/build.sh
        env:
          ARCH_PACKAGES: ${{ needs.prepare.outputs.arch_packages }}
          NOARCH_PACKAGES: ${{ needs.prepare.outputs.noarch_packages }}
          ARCH_MIN_DSM72_PACKAGES: ${{ needs.prepare.outputs.arch_min_dsm72_packages }}
          NOARCH_MIN_DSM72_PACKAGES: ${{ needs.prepare.outputs.noarch_min_dsm72_packages }}
          ARCH_MIN_DSM73_PACKAGES: ${{ needs.prepare.outputs.arch_min_dsm73_packages }}
          NOARCH_MIN_DSM73_PACKAGES: ${{ needs.prepare.outputs.noarch_min_dsm73_packages }}
          PUBLISH: ${{ github.event.inputs.publish }}
          API_KEY: ${{ secrets.PUBLISH_API_KEY }}
          PACKAGE_TO_PUBLISH: ${{ github.event.inputs.package }}
          # https://github.com/SynoCommunity/spksrc/wiki/Compile-and-build-rules
          GH_ARCH: ${{ matrix.arch }}
          BUILD_ERROR_FILE: /github/workspace/build_errors.txt
          BUILD_ERROR_LOGFILE: /github/workspace/build_log_errors.txt
          BUILD_UNSUPPORTED_FILE: /github/workspace/build_unsupported.txt
          BUILD_SUCCESS_FILE: /github/workspace/build_success.txt
          BUILD_BUDGET_SECONDS: ${{ env.BUILD_BUDGET_SECONDS }}
          BUILD_REMAINING_FILE: /github/workspace/build_remaining.txt
          BUILD_TIMEOUT_FILE: /github/workspace/build_timeout.txt

      # Check disk space after build completes (success or failure)
      - name: Disk Space After Build
        if: always()
        run: |
          echo "=== Disk Space after Build ==="
          df -h
          echo ""
          echo "=== Largest directories after build ==="
          du -h -d 2 /home/runner/work 2>/dev/null | sort -hr | head -20 || true
          echo ""
          
          # Create detailed disk usage report
          {
            echo "=========================================="
            echo "WORKSPACE DIRECTORIES"
            echo "=========================================="
            du -sh /home/runner/work/* 2>/dev/null || true
            
            echo ""
            echo "=========================================="
            echo "BUILD PACKAGES (with logs)"
            echo "=========================================="
            if [ -n "${{ needs.prepare.outputs.arch_packages }}" ] || [ -n "${{ needs.prepare.outputs.noarch_packages }}" ]; then
              for pkg in ${{ needs.prepare.outputs.arch_packages }} ${{ needs.prepare.outputs.noarch_packages }}; do
                if [ -d "spk/$pkg" ]; then
                  echo "Package: $pkg"
                  
                  # Show total package directory size
                  du -sh "spk/$pkg" 2>/dev/null || true
                  
                  # Show work directories if they exist
                  work_dirs=$(find "spk/$pkg" -maxdepth 2 -type d -name "work-*" 2>/dev/null)
                  if [ -n "$work_dirs" ]; then
                    echo "$work_dirs" | while read work_dir; do
                      du -sh "$work_dir" 2>/dev/null | sed 's/^/  /' || true
                    done
                  else
                    echo "  n/a	spk/$pkg/work-${{ matrix.arch }}"
                  fi
                  
                  # Show SPK file size if it exists
                  if ls packages/${pkg}*.spk 2>/dev/null; then
                    ls -lh packages/${pkg}*.spk 2>/dev/null | awk '{print "  " $5 "\tpackages/" $9}' || echo "  n/a	packages/${pkg}*.spk"
                  else
                    echo "  n/a	packages/${pkg}*.spk"
                  fi
                  
                  echo ""
                fi
              done
            else
              echo "No package list available - scanning for build directories..."
              find spk/ -type f -name "*.log" -o -name "work-*" 2>/dev/null | \
                xargs -I {} dirname {} | sort -u | \
                while read dir; do
                  du -sh "$dir" 2>/dev/null || true
                done
            fi
            
            echo ""
            echo "=========================================="
            echo "OVERALL DISK STATUS"
            echo "=========================================="
            df -h
            
          } > disk_usage_end.txt
          
          # Also display to console
          cat disk_usage_end.txt
          
          # Check if running low on disk space
          DISK_USAGE=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
          if [ "$DISK_USAGE" -gt 90 ]; then
            echo "::warning::Disk usage is at ${DISK_USAGE}% - build may have failed due to low disk space"
          fi
          
          # Emergency cleanup if disk is critically full (above 95%)
          if [ "$DISK_USAGE" -gt 95 ]; then
            echo "::error::CRITICAL: Disk usage is at ${DISK_USAGE}% - performing emergency cleanup"
            echo "disk_critical=true" >> $GITHUB_ENV
            
            # Clean up Docker to free space for diagnostics upload
            echo "Cleaning Docker system..."
            docker system prune -af --volumes 2>/dev/null || true
            
            # Remove large work directories except logs
            echo "Cleaning large build artifacts..."
            find . -type d -name "work-*" -exec sh -c 'find "$1" -type f ! -name "*.log" -delete' _ {} \; 2>/dev/null || true
            
            echo "Space after emergency cleanup:"
            df -h
          fi

      # Capture Docker logs when build fails
      - name: Capture Docker Logs on Failure
        if: failure()
        run: |
          echo "=== Attempting to capture docker logs ==="
          docker ps -a || true
          docker logs $(docker ps -aq | head -1) 2>&1 | tail -500 > docker_logs.txt || echo "Could not capture docker logs"

      # Collect build logs from each spk/<pkg>/ directory:
      # - top-level logs: build.log, build-<arch>.log, status-*.log, publish-*.log
      # - work-<arch>-<version>/ logs: config.log, CMakeOutput.log,
      #   meson-log.txt, ninja.log, etc. (preserving relative path for context)
      - name: Collect Build Logs from SPK directories
        if: always()
        run: |
          echo "=== Collecting build logs from spk directories ==="
          mkdir -p collected_logs

          # Patterns to collect from spk/<pkg>/ (top-level logs)
          TOP_LEVEL_PATTERNS=(
            "build-*.log"
            "status-*.log"
            "publish-*.log"
            "build.log"
          )

          # Patterns to collect recursively inside work-<arch>-<version>/ directories
          # These cover cmake, meson, autotools, ninja, cargo, etc.
          WORK_PATTERNS=(
            "build.log"
            "config.log"
            "configure.log"
            "CMakeCache.txt"
            "CMakeOutput.log"
            "CMakeError.log"
            "install_manifest.txt"
            "meson-log.txt"
            ".ninja_log"
            "ninja.log"
            "cargo.log"
            "*.log"
          )

          # Patterns to collect files specific at spk creation time
          SPK_PATTERNS=(
            ".copy_done"
            ".depend_done"
            ".icon_done"
            ".strip_done"
            ".stage0-tcvars_done"
            ".stage1-tcvars_done"
            ".stage1-tkvars_done"
            ".wheel_done"
            "INFO"
            "PLIST"
            "PACKAGE_ICON*"
            "WIZARD_UIFILES"
          )

          # Patterns to collect toolchain and toolkit environment
          TCVARS_PATTERNS=(
            "tc_vars*.mk"
            "tc_vars.cmake"
            "tc_vars.meson-*"
          )

          for pkg_dir in spk/*/; do
            pkg=$(basename "$pkg_dir")

            # ── Top-level package logs (build-<arch>.log, status-build.log …) ──
            for pattern in "${TOP_LEVEL_PATTERNS[@]}"; do
              find "$pkg_dir" -maxdepth 1 -type f -name "$pattern" -print0 2>/dev/null \
              | while IFS= read -r -d '' logfile; do
                  dest="collected_logs/${pkg}/$(basename "$logfile")"
                  mkdir -p "$(dirname "$dest")"
                  cp -p "$logfile" "$dest"
                  echo "Collected top-level: $logfile"
                done
            done

            find "$pkg_dir" -maxdepth 1 -type d -name "work-*" -print0 2>/dev/null \
            | while IFS= read -r -d '' work_dir; do
                work_name=$(basename "$work_dir")
                dest_dir="collected_logs/${pkg}/${work_name}"
                mkdir -p "$dest_dir"

                # ── Logs inside work-<arch>-<version>/ directories ──
                for pattern in "${WORK_PATTERNS[@]}"; do
                  find "$work_dir" -type f -name "$pattern" -print0 2>/dev/null
                done | sort -zu \
                | while IFS= read -r -d '' logfile; do
                    rel="${logfile#${work_dir}/}"
                    dest="${dest_dir}/${rel}"
                    mkdir -p "$(dirname "$dest")"
                    cp -p "$logfile" "$dest"
                    echo "Collected work log: $logfile"
                done

                # ── Toolchain/toolkit environment (tc_vars) ──
                for pattern in "${TCVARS_PATTERNS[@]}"; do
                  find "$work_dir" -maxdepth 1 \( -type f -o -type l \) -name "$pattern" -print0 2>/dev/null \
                  | while IFS= read -r -d '' f; do
                      cp -pL "$f" "$dest_dir/"
                      echo "Collected tcvars: $f"
                  done
                done

                # ── SPK creation state (spk cookies) ──
                for pattern in "${SPK_PATTERNS[@]}"; do
                  find "$work_dir" -maxdepth 1 \( -type f -o -type l \) -name "$pattern" -print0 2>/dev/null \
                  | while IFS= read -r -d '' f; do
                      cp -p "$f" "$dest_dir/"
                      echo "Collected spk state: $f"
                  done
                done

                # ── Dependency status cookies (.*_done minus SPK_PATTERNS) ──
                exclude_args=""
                for pattern in "${SPK_PATTERNS[@]}"; do
                  exclude_args="$exclude_args ! -name $pattern"
                done
                find "$work_dir" -maxdepth 1 -type f -name ".*_done" $exclude_args -print0 2>/dev/null \
                | while IFS= read -r -d '' f; do
                    cp -p "$f" "$dest_dir/"
                    echo "Collected status cookie: $f"
                done

                # ── library build state work-<arch>-<version>/install/.../lib/pkgconfig ──
                PKGCONFIG_DIR="$work_dir/install/var/packages/$pkg/target/lib/pkgconfig"
                if [ -d "$PKGCONFIG_DIR" ]; then
                  find "$PKGCONFIG_DIR" -type f -name "*.pc" -print0 2>/dev/null \
                  | while IFS= read -r -d '' f; do
                      rel="${f#${PKGCONFIG_DIR}/}"
                      dest="collected_logs/${pkg}/${work_name}/pkgconfig/${rel}"
                      mkdir -p "$(dirname "$dest")"
                      cp -p "$f" "$dest"
                      echo "Collected pkgconfig: $f"
                  done
                fi
              done
          done

          # Display summary of collected logs
          echo ""
          echo "=== Summary of collected logs ==="
          find collected_logs -type f | sort | while read f; do
            size=$(du -sh "$f" 2>/dev/null | cut -f1)
            echo "  $size  $f"
          done

          # Quick tail of the main build log for this arch
          echo ""
          echo "=== Last 100 lines of build-${{ matrix.arch }}.log ==="
          find spk/ -maxdepth 2 -type f -name "build-${{ matrix.arch }}.log" \
            -exec tail -100 {} \; 2>/dev/null \
            || echo "No build-${{ matrix.arch }}.log found"

          echo ""
          echo "=== status-build.log ==="
          find spk/ -maxdepth 2 -type f -name "status-build.log" \
            -exec cat {} \; 2>/dev/null \
            || echo "No status-build.log found"

          # Disk critical marker
          if [ "${{ env.disk_critical }}" == "true" ]; then
            echo "Build failed due to critical disk space (>95% full)" \
              > collected_logs/DISK_SPACE_CRITICAL.txt
            df -h >> collected_logs/DISK_SPACE_CRITICAL.txt
          fi

      - name: Build Status
        id: build_status
        # We need this status since we don't want to stop the build on errors.
        # Here we fail on build errors found in the build error file.
        uses: docker://ghcr.io/synocommunity/spksrc:latest
        with:
          entrypoint: ./.github/actions/build_status.sh
        env:
          BUILD_ERROR_FILE: /github/workspace/build_errors.txt
          BUILD_ERROR_LOGFILE: /github/workspace/build_log_errors.txt
          BUILD_UNSUPPORTED_FILE: /github/workspace/build_unsupported.txt
          BUILD_SUCCESS_FILE: /github/workspace/build_success.txt
          BUILD_REMAINING_FILE: /github/workspace/build_remaining.txt
          BUILD_TIMEOUT_FILE: /github/workspace/build_timeout.txt

      # Assemble flat diagnostics and the full collected_logs tree
      # into artifacts/diagnostics/ for upload.
      - name: Organize Diagnostic Logs
        if: always()
        continue-on-error: true
        run: |
          echo "=== Organizing diagnostic files ==="
          mkdir -p artifacts/diagnostics/collected_logs

          # Copy flat diagnostic files
          for file in disk_usage_start.txt disk_usage_end.txt docker_logs.txt \
                      build_errors.txt build_log_errors.txt \
                      build_unsupported.txt build_success.txt \
                      build_remaining.txt build_timeout.txt; do
            [ -f "$file" ] && cp -p "$file" "artifacts/diagnostics/$file"
          done

          # Copy collected_logs tree (preserves pkg/work-<arch>/ structure)
          if [ -d collected_logs ] && [ -n "$(ls -A collected_logs 2>/dev/null)" ]; then
            cp -pr collected_logs/. artifacts/diagnostics/collected_logs/
            echo "Copied $(find artifacts/diagnostics/collected_logs -type f | wc -l) log files"
          else
            echo "No collected logs to copy"
          fi

          echo "=== Diagnostic artifact contents ==="
          find artifacts/diagnostics -type f | sort | while read f; do
            size=$(du -sh "$f" 2>/dev/null | cut -f1)
            echo "  $size  $f"
          done

      # Upload per-arch diagnostics (retention: 1 day).
      # Consolidated across all archs by the consolidate-diagnostics job (retention: 30 days).
      - name: Upload Diagnostic Logs
        if: always()
        continue-on-error: true
        uses: actions/upload-artifact@v7
        with:
          name: diagnostics-${{ matrix.arch }}
          path: artifacts/diagnostics/
          if-no-files-found: ignore
          include-hidden-files: true
          retention-days: 1
          compression-level: 9

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: Packages for ${{ matrix.arch }}
          path: packages/*.spk
          if-no-files-found: ignore
          include-hidden-files: true

  consolidate-diagnostics:
    name: Consolidate Diagnostics
    needs: [gate, build]
    runs-on: ubuntu-latest
    if: always() && needs.gate.outputs.should_build == 'true'
    steps:
      - name: Download all diagnostic artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: diagnostics-*
          path: all-diagnostics
          merge-multiple: false

      - name: Upload consolidated diagnostics
        uses: actions/upload-artifact@v7
        with:
          name: Diagnostics
          path: all-diagnostics/**
          if-no-files-found: ignore
          include-hidden-files: true
          compression-level: 9