tests: add feature tagging workflow

.github/workflows/feature-tagging.yaml

@@ -0,0 +1,183 @@
+name: Feature Tagging
+
+on:
+  push:
+    branches: ["master"]
+
+  workflow_dispatch:
+    inputs:
+      features:
+        type: string
+        description: 'Comma-separated list of features to tag'
+        default: 'all'
+      maximum-reruns:
+        type: number
+        description: 'Maximum number of times to rerun failed spread tasks upon failure'
+        default: 3
+      run-all:
+        type: boolean
+        description: 'If true, will run all spread tests. If false, will only run tagging on changed spread tests in the last commit'
+        default: false
+
+jobs:
+  set-inputs:
+    runs-on: ubuntu-latest
+    outputs:
+      features: ${{ steps.step1.outputs.features }}
+      maximum-reruns: ${{ steps.step1.outputs.maximum-reruns }}
+      run-all: ${{ steps.step1.outputs.run-all }}
+    steps:
+      - name: Set inputs
+        run: |
+          echo "features=${{ inputs.features || 'all' }}" >> $GITHUB_OUTPUT
+          echo "maximum-reruns=${{ inputs.maximum-reruns || 3 }}" >> $GITHUB_OUTPUT
+          echo "run-all=${{ inputs.run-all || false }}" >> $GITHUB_OUTPUT
+
+  read-systems:
+    runs-on: ubuntu-latest
+    outputs:
+      fundamental-systems: ${{ steps.read-systems.outputs.fundamental-systems }}
+      non-fundamental-systems: ${{ steps.read-systems.outputs.non-fundamental-systems }}
+      nested-systems: ${{ steps.read-systems.outputs.nested-systems }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Read matrix file
+        id: read-systems
+        shell: bash
+        run: |
+          echo "fundamental-systems=$(jq -c . ./.github/workflows/fundamental-systems.json)" >> $GITHUB_OUTPUT
+          echo "non-fundamental-systems=$(jq -c . ./.github/workflows/non-fundamental-systems.json)" >> $GITHUB_OUTPUT
+          echo "nested-systems=$(jq -c . ./.github/workflows/nested-systems.json)" >> $GITHUB_OUTPUT
+
+  tag-features-fundamental:
+    uses: ./.github/workflows/spread-tests.yaml
+    needs: [set-inputs, read-systems]
+    name: "spread ${{ matrix.group }}"
+    with:
+      runs-on: '["self-hosted", "spread-enabled"]'
+      group: ${{ matrix.group }}
+      backend: ${{ matrix.backend }}
+      systems: ${{ matrix.systems }}  
+      tasks: ${{ matrix.tasks }}
+      rules: ${{ fromJSON(needs.set-inputs.outputs.run-all) && matrix.rules || 'feature-tagging.yaml' }}
+      is-fundamental: true
+      use-snapd-snap-from-master: true
+      spread-tag-features: ${{ needs.set-inputs.outputs.features }}
+      force-use-ruleset: ${{ ! fromJSON(needs.set-inputs.outputs.run-all) }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.read-systems.outputs.fundamental-systems) }}
+
+  tag-features-non-fundamental:
+    uses: ./.github/workflows/spread-tests.yaml
+    needs: [set-inputs, read-systems]
+    name: "spread ${{ matrix.group }}"
+    with:
+      runs-on: '["self-hosted", "spread-enabled"]'
+      group: ${{ matrix.group }}
+      backend: ${{ matrix.backend }}
+      systems: ${{ matrix.systems }}  
+      tasks: ${{ matrix.tasks }}
+      rules: ${{ fromJSON(needs.set-inputs.outputs.run-all) && matrix.rules || 'feature-tagging.yaml' }}
+      use-snapd-snap-from-master: true
+      spread-tag-features: ${{ needs.set-inputs.outputs.features }}
+      force-use-ruleset: ${{ ! fromJSON(needs.set-inputs.outputs.run-all) }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.read-systems.outputs.non-fundamental-systems) }}
+
+  tag-features-nested:
+    uses: ./.github/workflows/spread-tests.yaml
+    needs: [set-inputs, read-systems]
+    name: "spread ${{ matrix.group }}"
+    with:
+      runs-on: '["self-hosted", "spread-enabled"]'
+      group: ${{ matrix.group }}
+      backend: ${{ matrix.backend }}
+      systems: ${{ matrix.systems }}  
+      tasks: ${{ matrix.tasks }}
+      rules: ${{ fromJSON(needs.set-inputs.outputs.run-all) && matrix.rules || 'feature-tagging.yaml' }}
+      use-snapd-snap-from-master: true
+      spread-tag-features: ${{ needs.set-inputs.outputs.features }}
+      force-use-ruleset: ${{ ! fromJSON(needs.set-inputs.outputs.run-all) }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.read-systems.outputs.nested-systems) }}
+
+  re-run:
+    permissions:
+      actions: write
+    needs: [set-inputs, tag-features-fundamental, tag-features-non-fundamental, tag-features-nested]
+    # If the spread tests ended in failure, rerun the workflow up to maximum-reruns-1 times
+    if: failure() && fromJSON(github.run_attempt) < fromJSON(needs.set-inputs.outputs.maximum-reruns)
+    runs-on: ubuntu-latest
+    steps:
+      - env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+        run: gh workflow run rerun.yaml -F run_id=${{ github.run_id }}
+
+  create-reports:
+    needs: [set-inputs, tag-features-fundamental, tag-features-non-fundamental, tag-features-nested]
+    runs-on: ubuntu-latest
+    if: success() || fromJSON(github.run_attempt) >= fromJSON(needs.set-inputs.outputs.maximum-reruns)
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get generated data
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let page = 1;
+            let per_page = 100;
+            let allArtifacts = [];
+            let response;
+            do {
+              response = await github.rest.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.runId,
+                per_page: per_page,
+                page: page
+              });
+              allArtifacts = allArtifacts.concat(response.data.artifacts);
+              page++;
+            } while (response.data.artifacts.length === per_page);
+
+            let matchingArtifacts = allArtifacts.filter((artifact) => {
+              return artifact.name.startsWith(`feature-tags`);
+            });
+
+            for (let artifact of matchingArtifacts) {
+              let download = await github.rest.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: artifact.id,
+                archive_format: 'zip',
+              });
+              let fs = require('fs');
+              fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifact.name}.zip`, Buffer.from(download.data));
+              console.log(`Downloaded artifact: ${artifact.name}.zip`);
+            }
+      - name: Unzip artifacts
+        run: |
+          mkdir -p feature-tags-artifacts
+          find . -name "feature-tags*.zip" | while read filename; do 
+            unzip "$filename" -d "feature-tags-artifacts"
+          done
+
+      - name: Consolidate feature data
+        run: |
+          ./tests/lib/compose-features.py \
+            --dir "feature-tags-artifacts" \
+            --output "final-feature-tags" \
+            --replace-old-runs
+
+      - name: Upload feature data
+        uses: actions/upload-artifact@v4
+        with:
+          name: "feature-tags"
+          path: "final-feature-tags"

.github/workflows/rerun.yaml

@@ -0,0 +1,18 @@
+on:
+  workflow_dispatch:
+    inputs:
+      run_id:
+        required: true
+jobs:
+  rerun:
+    permissions: 
+      actions: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: rerun ${{ inputs.run_id }}
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh run watch ${{ inputs.run_id }} > /dev/null 2>&1
+          gh run rerun ${{ inputs.run_id }} --failed

.github/workflows/spread-tests.yaml

@@ -5,6 +5,11 @@ on:
         description: 'A json list of tags to indicate which runner to use'
         required: true
         type: string
+      force-use-ruleset:
+        description: 'If true, will force the use of the rulesset specified in the rules input'
+        required: false
+        default: false
+        type: boolean
       group:
         description: 'The name of the group of backends, systems, tests, and rules'
         required: true
@@ -43,12 +48,17 @@ on:
         description: 'Comma-separated list of experimental snapd features to enable with: snap set system "experimental.<feature-name>=true"'
         required: false
         type: string
+      spread-tag-features:
+        description: 'If specified, will tag the spread results with the specified features (comma-separated)'
+        required: false
+        type: string
 
 
 jobs:
   run-spread:
     env:
       SPREAD_EXPERIMENTAL_FEATURES: ${{ inputs.spread-experimental-features }}
+      SPREAD_TAG_FEATURES: ${{ inputs.spread-tag-features }}
 
     runs-on: ${{ fromJSON(inputs.runs-on) }}
     steps:
@@ -142,7 +152,11 @@ jobs:
             # The tests are just filtered when the change is a PR
             # When 'Run Nested' label is added in a PR, all the nested tests have to be executed
             TASKS_TO_RUN=""
-            if [ -z "${{ github.event.number }}" ] || [ "$RUN_NESTED" = 'true' ] || [ -z "${{ inputs.rules }}" ] || [ -z "$changes_param" ]; then
+            if [ "${{ inputs.force-use-ruleset }}" == "false" ] && \
+                ( [ -z "${{ github.event.number }}" ] || \
+                  [ "$RUN_NESTED" = 'true' ] || \
+                  [ -z "${{ inputs.rules }}" ] || \
+                  [ -z "$changes_param" ] ); then
                 for TASKS in ${{ inputs.tasks }}; do
                     TASKS_TO_RUN="$TASKS_TO_RUN $prefix:$TASKS"
                 done
@@ -292,14 +306,20 @@ jobs:
             exit 0
           fi
 
+          SPREAD_FLAGS='-no-debug-output -logs spread-logs'
+          if [ -n "$SPREAD_TAG_FEATURES" ]; then
+            SPREAD_FLAGS="$SPREAD_FLAGS -artifacts spread-artifacts"
+            echo "ARTIFACTS_FOLDER=spread-artifacts" >> $GITHUB_ENV
+          fi
+
           # Run spread tests
           # "pipefail" ensures that a non-zero status from the spread is
           # propagated; and we use a subshell as this option could trigger
           # undesired changes elsewhere
           echo "Running command: $SPREAD $RUN_TESTS"
           (
             set -o pipefail 
-            $SPREAD -no-debug-output -logs spread-logs $RUN_TESTS | \
+            $SPREAD $SPREAD_FLAGS $RUN_TESTS | \
               ./tests/lib/external/snapd-testing-tools/utils/log-filter $FILTER_PARAMS | \
               tee spread.log
           )
@@ -364,6 +384,23 @@ jobs:
             echo "TEST_FAILED=true" >> $GITHUB_ENV
           fi
 
+    - name: Analyze feature tags
+      if: always() && env.SPREAD_TAG_FEATURES != ''
+      run: |
+        ./tests/lib/compose-features.py \
+            --dir "${ARTIFACTS_FOLDER}/feature-tags" \
+            --output "feature-tags" \
+            --failed-tests "$(cat $FAILED_TESTS_FILE)" \
+            --run-attempt ${{ github.run_attempt }} \
+            --env-variables "SPREAD_EXPERIMENTAL_FEATURES=${SPREAD_EXPERIMENTAL_FEATURES},SPREAD_SNAPD_DEB_FROM_REPO=${SPREAD_SNAPD_DEB_FROM_REPO}"
+
+    - name: Upload feature tags
+      if: always() && env.SPREAD_TAG_FEATURES != ''
+      uses: actions/upload-artifact@v4
+      with:
+        name: "feature-tags-${{ inputs.group }}-${{ inputs.systems }}_${{ github.run_attempt }}"
+        path: "feature-tags"
+
     - name: Save spread test results to cache
       if: always()
       uses: actions/cache/save@v4

run-spread

@@ -28,5 +28,43 @@ if [ "$need_rebuild" = 1 ]; then
     echo "-- $(date) -- snapd snap rebuild complete"
 fi
 
-# Run spread
-SPREAD_USE_PREBUILT_SNAPD_SNAP=true exec spread "$@"
+if [ -z "$SPREAD_TAG_FEATURES" ]; then
+    SPREAD_USE_PREBUILT_SNAPD_SNAP=true exec spread "$@"
+else
+    WRITE_DIR="/tmp/features"
+    RUN_TESTS=("$@")
+    NUM_ATTEMPTS=${NUM_ATTEMPTS:-1}
+    export SPREAD_USE_PREBUILT_SNAPD_SNAP=true
+    mkdir -p "$WRITE_DIR"
+    for i in $(seq 1 "$NUM_ATTEMPTS"); do
+
+        spread -artifacts "${WRITE_DIR}"/features-artifacts -no-debug-output "${RUN_TESTS[@]}" | tee "${WRITE_DIR}/spread-logs.txt"
+
+        if [ -f "$WRITE_DIR"/spread-logs.txt ]; then
+            ./tests/lib/external/snapd-testing-tools/utils/log-parser "${WRITE_DIR}"/spread-logs.txt --output "${WRITE_DIR}"/spread-results.json
+            ./tests/lib/external/snapd-testing-tools/utils/log-analyzer list-reexecute-tasks "${RUN_TESTS[@]}" "${WRITE_DIR}"/spread-results.json > "${WRITE_DIR}"/failed-tests.txt
+        else
+            touch "${WRITE_DIR}/failed-tests.txt"
+        fi
+
+        ./tests/lib/compose-features.py \
+            --dir ${WRITE_DIR}/features-artifacts/feature-tags \
+            --output ${WRITE_DIR}/composed-feature-tags \
+            --failed-tests "$(cat ${WRITE_DIR}/failed-tests.txt)" \
+            --run-attempt "${i}"
+
+        if [ ! -s "${WRITE_DIR}/failed-tests.txt" ]; then
+            break
+        fi
+
+        mapfile RUN_TESTS < "${WRITE_DIR}"/failed-tests.txt
+    done
+
+    ./tests/lib/compose-features.py \
+        --dir ${WRITE_DIR}/composed-feature-tags \
+        --output ${WRITE_DIR}/final-feature-tags \
+        --replace-old-runs
+
+
+    echo "Your feature tags can be found in $WRITE_DIR/final-feature-tags"
+fi