spread-tests.yaml

name: Run spread tests

on:
  workflow_call:
    inputs:
      runs-on:
        description: 'A json list of tags to indicate which runner to use'
        required: true
        type: string
      group:
        description: 'The name of the group of backends, systems, tests, and rules'
        required: true
        type: string
      backend:
        description: 'The spread backend to use (for possible values, check spread.yaml). This cannot be empty'
        required: true
        type: string
      alternative-backend:
        description: 'The spread backend to use when the backend cannot be used'
        required: false
        type: string
      systems:
        description: 'The spread system(s) to use (for possible values, check spread.yaml). If more than one, separate them with a space. To run all, write ALL. To run none, write NONE'
        required: true
        type: string
      tasks:
        description: 'The spread tasks to run. It may be a space-separated list and may contain directories of many tasks or individual ones'
        required: true
        type: string
      rules:
        description: 'The rule .yaml file to use (found under tests/lib/spread/rules) for test discovery'
        required: true
        type: string
      is-fundamental:
        description: 'If true, then will mark results as from fundamental systems when uploading to Grafana'
        required: false
        type: boolean
        default: false
      use-snapd-snap-from-master:
        description: 'If true, will use the snapd snap built on the master branch'
        required: false
        type: boolean
        default: false
      spread-experimental-features:
        description: 'Comma-separated list of experimental snapd features to enable with: snap set system "experimental.<feature-name>=true"'
        required: false
        type: string
      spread-env:
        description: 'Extra environment variables to set before calling spread (format ENV1=VALUE1 ENV2=VALUE2) that are not covered by the other inputs'
        required: false
        type: string
      upload-artifacts:
        description: 'If true, will upload any artifacts that spread produces'
        type: boolean
        required: false
        default: false
      skip-tests:
        description: 'Space-separated list of regex expressions identifying tests to skip. As a regex, they can identify any part of the spread test like an entire suite or a single test. End a single test with $ to avoid inadvertently matching others.'
        required: false
        type: string


jobs:
  run-spread:
    env:
      SPREAD_EXPERIMENTAL_FEATURES: ${{ inputs.spread-experimental-features }}
      GH_TOKEN: ${{ github.token }}
      SPREAD_STORE_USER: ${{ secrets.SPREAD_STORE_USER }}
      SPREAD_STORE_PASSWORD: ${{ secrets.SPREAD_STORE_PASSWORD }}

    runs-on: ${{ fromJSON(inputs.runs-on) }}
    steps:

    - name: Verify inputs
      run: |
          if [ -z "${{ inputs.backend }}" ]; then
            echo "You must specify a backend."
            exit 1
          fi
          if [ -z "${{ inputs.systems }}" ]; then
            echo "You must specify a value for systems. If you want to run all, specify ALL. If you want to run none, specify NONE."
            exit 1
          fi

    - name: Cleanup job workspace
      id: cleanup-job-workspace
      run: |
          rm -rf "${{ github.workspace }}"
          mkdir "${{ github.workspace }}"

          # Kill any leftover spread processes
          echo "Leftover spread processes:"
          ps -e -o cmd | grep spread | grep -v grep || true
          killall -9 spread || true

    - name: Checkout code
      uses: actions/checkout@v6
      with:
        # spread uses tags as delta reference
        fetch-depth: 0

    - name: Set extra spread environment variables
      if: ${{ inputs.spread-env != '' }}
      run: |
          for env in ${{ inputs.spread-env }}; do
            echo "$env" >> $GITHUB_ENV
          done

    - name: Set PR labels as environment variables
      if: ${{ github.event.pull_request.number }}
      run: |
          labels=$(gh pr view ${{ github.event.pull_request.number }} --repo github.com/canonical/snapd --json labels | jq -r '.labels[].name')
          if grep -q '^Skip spread$' <<<$labels; then
            echo "SKIP_SPREAD_LABEL=true" >> $GITHUB_ENV
          fi
          if grep -q '^Run all$' <<<$labels; then
            echo "RUN_ALL_LABEL=true" >> $GITHUB_ENV
          fi
          if grep -q '^Run nested$' <<<$labels; then
            echo "RUN_NESTED_LABEL=true" >> $GITHUB_ENV
          fi

    - name: Get previous attempt
      id: get-previous-attempt
      run: |
        echo "previous_attempt=$(( ${{ github.run_attempt }} - 1 ))" >> $GITHUB_OUTPUT
      shell: bash

    - name: Download previous test results
      if: ${{ github.run_attempt > 1 }}
      uses: actions/download-artifact@v8
      continue-on-error: true
      with:
        name: "${{ github.job }}-results-${{ github.run_id }}-${{ inputs.group }}-${{ steps.get-previous-attempt.outputs.previous_attempt }}"
        path: "${{ github.workspace }}/.test-results"

    - name: Prepare test results env and vars
      id: prepare-test-results-env
      run: |
          # Create test results directories and save vars
          TEST_RESULTS_DIR="${{ github.workspace }}/.test-results"
          echo "TEST_RESULTS_DIR=$TEST_RESULTS_DIR" >> $GITHUB_ENV

          # Save the var with the failed tests file
          echo "FAILED_TESTS_FILE=$TEST_RESULTS_DIR/failed-tests" >> $GITHUB_ENV

          # Make sure the test results dirs are created
          # This step has to be after the cache is restored
          mkdir -p "$TEST_RESULTS_DIR"

    - name: Prepare nested env vars
      if: ${{ env.RUN_NESTED_LABEL == 'true' && startsWith(inputs.group, 'nested-') }}
      run: |
          echo "RUN_NESTED=true" >> "$GITHUB_ENV"

    - name: Get changed files
      id: changed-files
      uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6

    - name: Save changes files
      run: |
          CHANGED_FILES="${{ steps.changed-files.outputs.all_modified_files }}"
          echo "CHANGED_FILES=$CHANGED_FILES" >> $GITHUB_ENV
          echo "The changed files found are: $CHANGED_FILES"

    - name: Check failed tests to run
      if: ${{ env.RUN_ALL_LABEL != 'true' }}
      run: |
          if [ -s "$FAILED_TESTS_FILE" ]; then
              echo "FAILED_TESTS=true" >> $GITHUB_ENV
          else
              echo "FAILED_TESTS=false" >> $GITHUB_ENV
          fi

    - name: Setup for local run
      if: ${{ inputs.backend == 'garden' }}
      uses: ./.github/actions/setup-image-garden
      with:
        garden-system: ${{ inputs.systems }}

    - name: Setup run tests variable
      if: ${{ env.SKIP_SPREAD_LABEL != 'true' }}
      run: |
          get_new_tasks() {
            local prefix=$1
            local changes_param=$2
            # The tests are just filtered when the change is a PR
            # When 'Run Nested' label is added in a PR, all the nested tests have to be executed
            TASKS_TO_RUN=""
            INPUT_TASKS="$(echo "${{ inputs.tasks }}" | tr '\n' ' ')"
            if [ -z "${{ github.event.number }}" ] || [ "$RUN_NESTED" = 'true' ] || [ -z "${{ inputs.rules }}" ] || [ -z "$changes_param" ]; then
                for TASKS in $INPUT_TASKS; do
                    TASKS_TO_RUN="$TASKS_TO_RUN $prefix:$TASKS"
                done
            else
                NEW_TASKS="$(./tests/lib/external/snapd-testing-tools/utils/spread-filter -r ./tests/lib/spread/rules/${{ inputs.rules }}.yaml -p "$prefix" $changes_param -t "$INPUT_TASKS")"
                TASKS_TO_RUN="$NEW_TASKS"
            fi
            echo "$TASKS_TO_RUN"
          }

          # Determine if the alternative backend has to be used for the current group
          # Alternative backends usage depends on the value stored in the file USE_ALTERNATIVE_BACKEND,
          # where it is defined the backend to use when images are ready in both google and openstack.
          USE_ALTERNATIVE_BACKEND="$(curl -s https://storage.googleapis.com/snapd-spread-tests/snapd-tests/ci/use_alternative_backend.json)"
          SPREAD_BACKEND="${{ inputs.backend }}"
          if [ -n "${{ inputs.alternative-backend }}" ] && [ $(jq -r ".\"${{ inputs.group }}\"" <<< "$USE_ALTERNATIVE_BACKEND") == true ]; then
              SPREAD_BACKEND="${{ inputs.alternative-backend }}"
          fi
          echo SPREAD_BACKEND="$SPREAD_BACKEND" >> $GITHUB_ENV

          CHANGES_PARAM=""
          for CHANGE in $CHANGED_FILES; do
              CHANGES_PARAM="$CHANGES_PARAM -c $CHANGE"
          done

          TESTS_TO_RUN=""
          if [ "$FAILED_TESTS" = 'true' ]; then
              TESTS_TO_RUN="$(cat "$FAILED_TESTS_FILE")"
          elif [ "${{ inputs.systems }}" = "ALL" ]; then
              TESTS_TO_RUN=$(get_new_tasks "$SPREAD_BACKEND" "$CHANGES_PARAM")
          elif [ "${{ inputs.systems }}" != "NONE" ]; then
              for SYSTEM in ${{ inputs.systems }}; do
                NEW_TASKS="$(get_new_tasks "$SPREAD_BACKEND:$SYSTEM" "$CHANGES_PARAM")"
                if [ -z "$TESTS_TO_RUN" ]; then
                    TESTS_TO_RUN="$NEW_TASKS"
                else
                    TESTS_TO_RUN="$TESTS_TO_RUN $NEW_TASKS"
                fi
              done
          fi

          RUN_TESTS_FILE="${{ github.workspace }}/run-tests-file.txt"
          echo "RUN_TESTS_FILE=$RUN_TESTS_FILE" >> $GITHUB_ENV
          if [ -n "$TESTS_TO_RUN" ]; then
              echo "$TESTS_TO_RUN" > "$RUN_TESTS_FILE"
              echo "RUN_TESTS=true" >> $GITHUB_ENV
          else
              echo "RUN_TESTS=false" >> $GITHUB_ENV
              printf "" > "$RUN_TESTS_FILE"
          fi

    - name: Check un-skip tests in PR description
      if: ${{ github.event.pull_request.number }}
      run: |
          body="$(gh pr view ${{ github.event.pull_request.number }} --json body --jq .body)"

          # Grab "unskip:" and the following lines that start with "-".
          unskip_yaml="$(
              grep -ozP '(?ms)^[[:space:]]*unskip:[[:space:]]*\n(?:[[:space:]]*-[^\n]*\n?)*' <<<"$body" |
                  tr '\0' '\n' || true
          )"
          if [ -z "$unskip_yaml" ]; then
              exit 0
          fi

          mapfile -t unskip_tests < <(printf '%s\n' "$unskip_yaml" | yq -r '.unskip // [] | .[]?')
          UNSKIP_TESTS="${unskip_tests[*]}"

          if [ -z "$UNSKIP_TESTS" ]; then
              exit 0
          fi

          echo "UNSKIP_TESTS=$UNSKIP_TESTS" >> "$GITHUB_ENV"
          

    - name: Checkout snapd-testing-skip
      if: ${{ env.SKIP_SPREAD_LABEL != 'true' && env.RUN_TESTS == 'true' }}
      uses: actions/checkout@v6
      with:
          repository: canonical/snapd-testing-skip
          path: snapd-testing-skip

    - name: Filter out tests to skip
      if: ${{ env.SKIP_SPREAD_LABEL != 'true' && env.RUN_TESTS == 'true' }}
      run: |
          spread_list="$(spread -list $(cat "$RUN_TESTS_FILE") 2>&1 || true)"
          if grep -q "nothing matches provider filter" <<< "$spread_list"; then
            echo "No tests to run"
            echo "RUN_TESTS=false"  >> $GITHUB_ENV
            printf "" > "$RUN_TESTS_FILE"
            exit 0
          fi
          if [ -n "${{ inputs.skip-tests }}" ]; then
              for test in ${{ inputs.skip-tests }}; do
                  spread_list=$(echo "$spread_list" | tr ' ' '\n' | grep -vE "$test" | tr '\n' ' ')
              done
              echo "$spread_list" > "$RUN_TESTS_FILE"
          fi

          shopt -s nullglob
          SNAPD_TESTS_SKIP=
          for file in snapd-testing-skip/skip/*.yaml; do
              expire=$(yq -r '."skip-until"' "$file")
              today=$(date "+%Y-%m-%d")
              system=$(yq -r '."spread-system" // ".*"' "$file")
              backend=$(yq -r '."spread-backend" // ".*"' "$file")
              if [[ "$expire" > "$today" ]] && grep -q "$system" <<<"${{ inputs.systems }}" && grep -q "^$backend$" <<<"$SPREAD_BACKEND"; then
                  task=$(yq -r '."spread-task"' "$file")
                  IFS=' ' read -ra unskip_arr <<< "${UNSKIP_TESTS:-}"
                  for unskip_test in "${unskip_arr[@]}"; do
                      # Validate format: must be backend:system:task
                      if [[ ! "$unskip_test" =~ ^[^:]+:[^:]+:.+$ ]]; then
                          echo "Ignoring malformed unskip entry: $unskip_test"
                          continue
                      fi
                      unskip_backend="${unskip_test%%:*}"
                      unskip_rest="${unskip_test#*:}"
                      unskip_system="${unskip_rest%%:*}"
                      unskip_task="${unskip_rest#*:}"
                      if [[ "$unskip_task" == "$task" ]] && grep -q "$unskip_system" <<<"${{ inputs.systems }}" && grep -q "^$unskip_backend$" <<<"$SPREAD_BACKEND"; then
                          echo "Ignoring skip rule for $task due to unskip entry: $unskip_test"
                          continue 2
                      fi
                  done

                  SNAPD_TESTS_SKIP="${SNAPD_TESTS_SKIP} $(echo "$spread_list" | tr ' ' '\n' | grep -E "$task(:.*)?$" | tr '\n' ' ')"
                  spread_list=$(echo "$spread_list" | tr ' ' '\n' | grep -vE "$task(:.*)?$" | tr '\n' ' ')
                  echo "$spread_list" > "$RUN_TESTS_FILE"
              fi
          done

          if [ -z "${spread_list// /}" ]; then
              echo "RUN_TESTS=false" >> $GITHUB_ENV
              printf "" > "$RUN_TESTS_FILE"
          fi
          echo "The following list of tests will run:"
          cat "$RUN_TESTS_FILE"

          if [ -n "$SNAPD_TESTS_SKIP" ]; then
              echo "$SNAPD_TESTS_SKIP" > skipped_tests_list.txt
          fi

    - name: Upload skipped tests list
      uses: actions/upload-artifact@v7
      with:
        name: "skipped-tests-list-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.group }}"
        path: "skipped_tests_list.txt"
        if-no-files-found: ignore

    - name: Check if system is required
      if: always()
      env:
        GH_TOKEN: ${{ github.token }}
      run: |
        required=$(gh api repos/${{ github.repository }}/rules/branches/master | \
          jq -r '.[] | select(.type=="required_status_checks") | .parameters.required_status_checks' | \
          grep -q "spread ${{ inputs.group }} /" && echo "req" || echo "not-req")
        echo REQUIRED="$required" >> $GITHUB_ENV

    - name: Setup grafana parameters
      if: ${{ env.SKIP_SPREAD_LABEL != 'true' }}
      run: |
          # Configure parameters to filter logs (these logs are sent read by grafana agent)
          CHANGE_ID="${{ github.event.number }}"
          if [ -z "$CHANGE_ID" ]; then
            CHANGE_ID="main"
          fi
          fundamental=""
          if [ "${{ inputs.is-fundamental }}" = 'true' ]; then
            fundamental="fund"
          fi
          FILTERED_LOG_FILE="spread_${CHANGE_ID}_${fundamental}_${REQUIRED}_n${{ github.run_attempt }}.filtered.log"
          # The log-filter tool is used to filter the spread logs to be stored
          echo FILTER_PARAMS="-o $FILTERED_LOG_FILE -e Debug -e WARNING: -f Failed=NO_LINES -f Error=NO_LINES"  >> $GITHUB_ENV
          echo FILTERED_LOG_FILE="$FILTERED_LOG_FILE"  >> $GITHUB_ENV

          # Add start line to filtered log
          echo "GRAFANA START: pr ${CHANGE_ID} attempt ${{ github.run_attempt }} run ${{ github.run_id }} group ${{ inputs.group }}" > "$FILTERED_LOG_FILE"

    - name: Download and extract pre-built packages if they exist
      if: ${{ env.SKIP_SPREAD_LABEL != 'true' && env.RUN_TESTS == 'true' }}
      env:
        SYSTEMS: ${{ inputs.systems }}
      uses: actions/github-script@v9
      with:
        script: |
          let fs = require('fs');
          let path = require('path');
          let os = require('os');
          let child_process = require('child_process');
          let systems = process.env.SYSTEMS.split(' ');

          let page = 1;
          let per_page = 100;
          let allArtifacts = [];
          let response;
          do {
            response = await github.rest.actions.listWorkflowRunArtifacts({
              owner: context.repo.owner,
              repo: context.repo.repo,
              run_id: context.runId,
              per_page: per_page,
              page: page
            });
            allArtifacts = allArtifacts.concat(response.data.artifacts);
            page++;
          } while (allArtifacts.length < response.data.total_count);
          
          for (let system of systems) {
            let artifactName = `package-${system}`;
            const match = system.match(/^ubuntu-core-(\d+)(-arm)?-64$/);
            if (match) {
              const version = match[1];
              const arch = match[2] || '';
              artifactName = `package-ubuntu-${version}.04${arch}-64`;
            }
            let artifact = allArtifacts.find(a => a.name === artifactName);
            if (!artifact) {
              console.log(`Artifact not found for system: ${system}`);
              continue;
            }
            console.log(`Found artifact "${artifactName}" with id ${artifact.id}, downloading...`);

            const download = await github.rest.actions.downloadArtifact({
              owner: context.repo.owner,
              repo: context.repo.repo,
              artifact_id: artifact.id,
              archive_format: 'zip',
            });

            const artifactZipPath = path.join(process.env.GITHUB_WORKSPACE, `${artifactName}.zip`);
            fs.writeFileSync(artifactZipPath, Buffer.from(download.data));

            const extractDir = path.join(process.env.GITHUB_WORKSPACE, 'built-pkgs', system);
            fs.mkdirSync(extractDir, { recursive: true });
            child_process.execSync(`unzip -q -o ${artifactZipPath} -d ${extractDir}`);

            fs.unlinkSync(artifactZipPath);
          }

    - name: Set whether to use pre-built packages in spread
      run: |
        SPREAD_USE_PREBUILT_PACKAGES=true
        for system in ${{ inputs.systems }}; do
          if ! [ -d built-pkgs/"$system" ]; then
            SPREAD_USE_PREBUILT_PACKAGES=false
            echo "Disabling prebuilt packages due to missing $system directory"
          fi
        done
        echo "SPREAD_USE_PREBUILT_PACKAGES=$SPREAD_USE_PREBUILT_PACKAGES" >> $GITHUB_ENV

    - name: Download snapd snap
      if: ${{ env.SKIP_SPREAD_LABEL != 'true' && env.RUN_TESTS == 'true' }}
      uses: ./.github/actions/download-snapd-snap
      with:
        use-master-push: ${{ inputs.use-snapd-snap-from-master }}
        is-amd64: ${{ !contains(inputs.group, '-arm64') }}
        is-arm64: ${{ contains(inputs.group, '-arm64') }}
        is-fips: ${{ contains(inputs.group, '-fips') }}

    - name: Run spread tests
      if: ${{ env.SKIP_SPREAD_LABEL != 'true' && env.RUN_TESTS == 'true' }}
      env:
          SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
      run: |
          # Register a problem matcher to highlight spread failures
          echo "::add-matcher::.github/spread-problem-matcher.json"

          SPREAD=spread
          if [[ "${{ inputs.group }}" =~ nested- ]]; then
            export NESTED_BUILD_SNAPD_FROM_CURRENT=true
            export NESTED_ENABLE_KVM=true
          fi

          export SPREAD_USE_PREBUILT_SNAPD_SNAP=true

          # This could be the case when either there are not systems for a group or
          # the list of tests to run is empty
          TESTS_TO_RUN="$(cat "$RUN_TESTS_FILE")"
          if [ -z "${TESTS_TO_RUN// /}" ]; then
            echo "No tests to run, exiting..."
            exit 0
          fi

          if [[ "$TESTS_TO_RUN" =~ amazon-linux-2023 ]]; then
              # Amazon Linux 2023 has no xdelta, however we cannot disable
              # xdelta on a per-target basis as it's used in the repack section
              # of spread.yaml, which is shared by all targets, so all systems
              # in this batch will not use delta for transferring project data
              echo "Disabling xdelta support"
              export NO_DELTA=1
          fi

          case "${{ inputs.group }}" in
              ubuntu-deb)
                  echo "Setting up ubuntu with deb execution"
                  export SPREAD_SNAP_REEXEC=0
                  export SPREAD_SNAPD_DEB_FROM_REPO=false
                  ;;
          esac

          SPREAD_FLAGS='-no-debug-output -logs spread-logs -json results.json'
          if [ "${{ inputs.upload-artifacts }}" = true ]; then
            SPREAD_FLAGS="$SPREAD_FLAGS -artifacts spread-artifacts"
            echo "ARTIFACTS_FOLDER=spread-artifacts" >> $GITHUB_ENV
          fi

          # Run spread tests
          # "pipefail" ensures that a non-zero status from the spread is
          # propagated; and we use a subshell as this option could trigger
          # undesired changes elsewhere
          echo "Running command: $SPREAD $TESTS_TO_RUN"
          (
            set -o pipefail 
            $SPREAD $SPREAD_FLAGS $TESTS_TO_RUN | \
              ./tests/lib/external/snapd-testing-tools/utils/log-filter $FILTER_PARAMS | \
              tee spread.log
          )

    - name: Discard spread workers
      if: always()
      run: |
        # Kill any leftover spread processes
        echo "Leftover spread processes:"
        ps -e -o cmd | grep spread | grep -v grep || true
        killall -9 spread || true

        shopt -s nullglob;
        for r in .spread-reuse.*.yaml; do
          spread -discard -reuse-pid="$(echo "$r" | grep -o -E '[0-9]+')";
        done

    - name: Upload spread logs
      if: always()
      uses: actions/upload-artifact@v7
      with:
        name: "spread-logs-${{ inputs.group }}-${{ inputs.systems }}"
        path: "spread-logs/*.log"
        if-no-files-found: ignore
  
    - name: Upload spread results file
      if: always()
      uses: actions/upload-artifact@v7
      with:
        name: "spread-results-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.group }}"
        path: "results.json"
        if-no-files-found: ignore

    - name: Report spread errors
      if: always()
      run: |
        if [ -e results.json ]; then
            echo "Adding to the filtered log any spread errors found"
            ACTIONS_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
            while IFS= read -r line; do
                if [ ! -z "$line" ]; then
                    echo "Adding failed test line to filtered log"
                    echo "GRAFANA FAILED: $line $ACTIONS_URL" | tee -a "$FILTERED_LOG_FILE"
                fi
            done <<< $(jq -r '.items[] | select(.level == "task" and .success == false and .skipped == false and .verb != "checking") | "\(.verb) \(.backend):\(.system):\(.name)\(if .variant != "" then ":\(.variant)" else "" end)"' results.json | sort -u)
        else
            echo "No spread log found, skipping errors reporting"
        fi

    - name: Analyze spread test results
      if: always()
      run: |
          if [ -f results.json ]; then
              echo "Saving the tests to be re-executed"
              # Tests to be re-executed are:
              # - Aborted tests (level is empty and aborted is true)
              # - Failed tasks (level is task and success is false)
              # - Need to be ignored the checking verb (checking skip condition)
              # The format is backend:system:name:variant (variant is optional)
              # Duplicates are removed and the list is sorted
              # The list is saved in the FAILED_TESTS_FILE
              jq -r '.items[] | select(.level == "task" and .success == false and .skipped == false and .verb != "checking") | "\(.backend):\(.system):\(.name)\(if .variant != "" then ":\(.variant)" else "" end)"' results.json | sort -u | xargs > "$FAILED_TESTS_FILE"

              if [ -n "${{ inputs.upload-artifacts }}" ] && \
                  [ -d "$ARTIFACTS_FOLDER" ] && \
                  [ -n "$(jq -r '.items[] | select(.verb== "restoring" and .success == false)' results.json)" ]; then

                  # Restore encountered some errors, so check the retrieved data for any missing tests.
                  # If any data is missing, add the corresponding test to the list of failed tests.
                  for test in $(cat "$RUN_TESTS_FILE"); do
                      dir="$(find $ARTIFACTS_FOLDER -name "$(sed 's/\//--/g' <<<$test)" -type d 2>/dev/null || true)"
                      if [ -z "$dir" ] && ! grep -q "$test" "$FAILED_TESTS_FILE"; then
                          printf "%s %s\n" "$(< $FAILED_TESTS_FILE)" "$test" > "$FAILED_TESTS_FILE"
                      fi
                  done
              fi

              echo "List of failed tests saved"
              cat "$FAILED_TESTS_FILE"
          else
              echo "No spread log found, saving empty list of failed tests"
              touch "$FAILED_TESTS_FILE"
          fi
          
          # Set the TEST_FAILED variable if there are any failed tests
          if [ -n "$(cat $FAILED_TESTS_FILE)" ]; then
            echo "TEST_FAILED=true" >> $GITHUB_ENV
          fi

    - name: Prepare artifact upload
      if: always() && fromJSON(inputs.upload-artifacts)
      run: |
        if [ -z "${ARTIFACTS_FOLDER}" ]; then
            echo "No artifacts folder found"
            exit 0
        fi

        echo "${{ github.run_attempt }}" > "${ARTIFACTS_FOLDER}/run-attempt.txt"
        cp "$FAILED_TESTS_FILE" "${ARTIFACTS_FOLDER}/failed-tests.txt"
        echo "SPREAD_EXPERIMENTAL_FEATURES=${SPREAD_EXPERIMENTAL_FEATURES}" > "${ARTIFACTS_FOLDER}/env-variables.txt"
        echo "SPREAD_SNAPD_DEB_FROM_REPO=${SPREAD_SNAPD_DEB_FROM_REPO}" >> "${ARTIFACTS_FOLDER}/env-variables.txt"
        tar -czf "spread-artifacts-${{ inputs.group }}-${{ inputs.systems }}_${{ github.run_id }}_${{ github.run_attempt }}.tar.gz" "$ARTIFACTS_FOLDER"

    - name: Upload artifacts
      if: always() && fromJSON(inputs.upload-artifacts)
      uses: actions/upload-artifact@v7
      with:
        name: "spread-artifacts-${{ inputs.group }}-${{ inputs.systems }}_${{ github.run_id }}_${{ github.run_attempt }}"
        path: "spread-artifacts-${{ inputs.group }}-${{ inputs.systems }}_${{ github.run_id }}_${{ github.run_attempt }}.tar.gz"
        if-no-files-found: ignore

    - name: Upload failed tests
      if: always() && env.TEST_FAILED == 'true'
      uses: actions/upload-artifact@v7
      with:
        name: "${{ github.job }}-results-${{ github.run_id }}-${{ inputs.group }}-${{ github.run_attempt }}"
        path: ${{ env.FAILED_TESTS_FILE }}

    - name: Send results to Test-Predictor
      if: always()
      continue-on-error: true
      run: |
        if [ -f results.json ]; then
            echo "Sending results to Test-Predictor"

            # Get the job ID of the current workflow run by matching job name and matrix group
            JOB_ID=$(gh api --paginate repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs --jq ".jobs[] | select(.name | contains(\"${{ inputs.group }}\") and contains(\"${{ github.job }}\")) | .id" | head -n 1)

            echo "Sending results.json with run_id: ${{ github.run_id }}, job_id: $JOB_ID and attempt: ${{ github.run_attempt }}"
            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://test-predictor.canonical.com:5000/ingest \
              -F "file=@./results.json" \
              -F "attempt=${{ github.run_attempt }}" \
              -F "job_id=$JOB_ID" \
              -F "run_id=${{ github.run_id }}")

            if [[ ! $HTTP_CODE =~ ^2[0-9]{2}$ ]]; then
                echo "Request failed with status $HTTP_CODE"
                exit 1
            fi
            echo "Results sent successfully"

            echo "Getting predictions for failed tests"
            wget -q -O predict https://raw.githubusercontent.com/canonical/snapd-testing/master/ai/test-predictor/src/client/predict
            chmod +x predict
            ./predict results.json
        else
            echo "No spread results log found, skipping sending results to Test-Predictor"
        fi