actions(build): unify test results and summaries into a single tarball for each test run

Instead of uploading our failure summaries and test results files
directly to Github Actions storage, instead we create a unified gzipped
tarball with all of the files. This is useful because

- The Summary tab of the workflow is already polluted with too much
  information. I really only want to see what failed. That doesn't
  completed address that but it's a first step.

- The Summary tab has loads of files attached that aren't the actual
  artifacts. This actually impacts CRT further down the line when it
  downloads these files and uploads them to Artifactory. We're hoping
  that by creating our own archive that might help an issue we've been
  seeing lately.

- When I want debug/test information for a test scenario I can almost
  get everything in a single bundle. Future enhancement might be to also
  include the enos debug object files in this too.

Signed-off-by: Ryan Cragun <me@ryan.ec>
Co-authored-by: Ryan Cragun <me@ryan.ec>

Author: hc-github-team-secure-vault-core

.github/workflows/build.yml

@@ -731,14 +731,48 @@ jobs:
           fi
       - id: slackbot-webhook-url
         run:
-          echo "slackbot-webhook-url=${{ needs.setup.outputs.is-ent-repo != 'true' && secrets.FEED_VAULT_CI_OFFICIAL_WEBHOOK_URL || steps.secrets.outputs.slackbot-webhook-url }}" >> "$GITHUB_OUTPUT"
+          echo "slackbot-webhook-url=${{ needs.setup.outputs.is-ent-repo != 'true' && secrets.FEED_VAULT_CI_OFFICIAL_WEBHOOK_URL || steps.secrets.outputs.slackbot-webhook-url }}" | tee -a "$GITHUB_OUTPUT"
       - if: ${{ needs.setup.outputs.is-fork == 'false' }}
-        name: Download failure summaries
+        name: Download test results tarballs
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          pattern: failure-summary-*.md
-          path: failure-summaries
+          pattern: test-results_*.tar.gz
+          path: test-results-tarballs
           merge-multiple: true
+      - if: ${{ needs.setup.outputs.is-fork == 'false' }}
+        name: Extract test results and failure summaries
+        run: |
+          mkdir -p failure-summaries
+          extraction_errors=0
+
+          if [ -d "test-results-tarballs" ]; then
+            for tarball in test-results-tarballs/*.tar.gz; do
+              if [ -f "$tarball" ]; then
+                temp_extract_dir=$(mktemp -d)
+
+                # Extract tarball with error checking
+                if ! tar -xzf "$tarball" -C "$temp_extract_dir"; then
+                  echo "::error file=$(basename "$tarball")::Failed to extract tarball"
+                  extraction_errors=$((extraction_errors + 1))
+                  rm -rf "$temp_extract_dir"
+                  continue
+                fi
+
+                # Copy failure summaries if they exist
+                if [ -d "$temp_extract_dir/failure_summary" ]; then
+                  if ls "$temp_extract_dir/failure_summary"/*.md &>/dev/null; then
+                    cp "$temp_extract_dir/failure_summary"/*.md failure-summaries/
+                  fi
+                fi
+
+                rm -rf "$temp_extract_dir"
+              fi
+            done
+
+            if [ $extraction_errors -gt 0 ]; then
+              echo "::error::Failed to extract $extraction_errors tarball(s)"
+            fi
+          fi
       - if: ${{ needs.setup.outputs.is-fork == 'false' }}
         id: prepare-failure-summary
         name: Prepare failure summary

.github/workflows/test-run-enos-scenario-matrix.yml

@@ -342,43 +342,17 @@ jobs:
           if-no-files-found: ignore
         continue-on-error: true
 
-      - name: Upload Test Results
+      - name: Prepare Test Results and Failure Summaries
         if: always()
-        id: upload_test_results
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: ${{ steps.prepare_scenario.outputs.test_results_artifact_name }}
-          path: /tmp/vault_test_results_*.json
-          retention-days: 7
-          if-no-files-found: ignore
-        continue-on-error: true
-
-      - name: Upload JUnit Test Results
-        if: always()
-        id: upload_junit_results
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: ${{ steps.prepare_scenario.outputs.junit_results_artifact_name }}
-          path: /tmp/vault_test_results_*.xml
-          retention-days: 7
-          if-no-files-found: ignore
-        continue-on-error: true
-
-      - name: Check for test results
-        if: always()
-        id: check_test_results
+        id: prepare_test_summary
         continue-on-error: true
         run: |
-          if find /tmp -maxdepth 1 -name 'vault_test_results_*.json' -type f -print -quit 2>/dev/null | grep -q .; then
-            echo "has_results=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "has_results=false" >> "$GITHUB_OUTPUT"
+          if ! find /tmp -maxdepth 1 -name 'vault_test_results_*.json' -type f -print -quit 2>/dev/null | grep -q .; then
+            echo "failure_summary_created=false" | tee -a "$GITHUB_OUTPUT"
+            echo "::warning::No test results found in /tmp/vault_test_results_*.json"
+            exit 0
           fi
 
-      - name: Prepare Test Results Summary
-        if: always() && steps.check_test_results.outputs.has_results == 'true'
-        continue-on-error: true
-        run: |
           # Find the most recent JSON test results file
           json_file=$(find /tmp -maxdepth 1 -name 'vault_test_results_*.json' -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -n1 | cut -d' ' -f2-)
 
@@ -396,6 +370,9 @@ jobs:
             passed_tests=$(jq -r 'select(.Action == "pass") | select(.Test != null)' "$json_file" | jq -s 'length')
             failed_tests=$(jq -r 'select(.Action == "fail") | select(.Test != null)' "$json_file" | jq -s 'length')
 
+            # Output for use in tarball creation
+            echo "failure_summary_created=true" | tee -a "$GITHUB_OUTPUT"
+
             # Create step summary for this specific scenario
             {
               echo "## Test Results for ${{ matrix.scenario.id.filter }}"
@@ -414,19 +391,66 @@ jobs:
                 cat "${failure_summary_file}"
                 echo ""
               fi
+            } | tee -a "$GITHUB_STEP_SUMMARY"
+          else
+            echo "failure_summary_created=false" | tee -a "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create test results tarball
+        if: always()
+        id: create_test_results_tarball
+        continue-on-error: true
+        run: |
+          mkdir -p /tmp/test-artifacts/test_results
+          mkdir -p /tmp/test-artifacts/junit_results
+          mkdir -p /tmp/test-artifacts/failure_summary
 
-              echo "📊 Full test results available in artifacts: \`${{ steps.prepare_scenario.outputs.test_results_artifact_name }}\`"
-            } >> "$GITHUB_STEP_SUMMARY"
+          if ls /tmp/vault_test_results_*.json 1> /dev/null 2>&1; then
+            cp /tmp/vault_test_results_*.json /tmp/test-artifacts/test_results/
+          fi
+
+          if ls /tmp/vault_test_results_*.xml 1> /dev/null 2>&1; then
+            cp /tmp/vault_test_results_*.xml /tmp/test-artifacts/junit_results/
+          fi
+
+          failure_summary_file="${{ steps.prepare_scenario.outputs.failure_summary_artifact_name }}"
+          if [ "${{ steps.prepare_test_summary.outputs.failure_summary_created }}" == "true" ] && [ -f "$failure_summary_file" ]; then
+            cp "$failure_summary_file" /tmp/test-artifacts/failure_summary/
+          fi
+
+          if [ -n "$(ls -A /tmp/test-artifacts/test_results 2>/dev/null)" ] || \
+             [ -n "$(ls -A /tmp/test-artifacts/junit_results 2>/dev/null)" ] || \
+             [ -n "$(ls -A /tmp/test-artifacts/failure_summary 2>/dev/null)" ]; then
+            scenario_safe_name=$(echo "${{ matrix.scenario.id.filter }}" | sed -e 's/ /_/g' | sed -e 's/:/=/g')
+            # Truncate to 215 chars to leave room for .tar.gz suffix and buffer
+            tarball_name="test-results_${scenario_safe_name:0:215}.tar.gz"
+
+            if tar -czf "/tmp/${tarball_name}" -C /tmp/test-artifacts .; then
+              {
+                echo "tarball_created=true"
+                echo "tarball_name=${tarball_name}"
+                echo "tarball_path=/tmp/${tarball_name}"
+              } | tee -a "$GITHUB_OUTPUT"
+
+              # Add artifact reference to step summary
+              echo "" | tee -a "$GITHUB_STEP_SUMMARY"
+              echo "Full test results available in artifact: \`${tarball_name}\`" | tee -a "$GITHUB_STEP_SUMMARY"
+            else
+              echo "::error::Failed to create tarball"
+              echo "tarball_created=false" | tee -a "$GITHUB_OUTPUT"
+            fi
           else
-            echo "⚠️ No test results found in /tmp/vault_test_results_*.json" >> "$GITHUB_STEP_SUMMARY"
+            echo "tarball_created=false" | tee -a "$GITHUB_OUTPUT"
           fi
 
-      - name: Upload Failure Summary
-        if: always() && steps.check_test_results.outputs.has_results == 'true'
+      - name: Upload test results tarball
+        if: always() && steps.create_test_results_tarball.outputs.tarball_created == 'true'
+        id: upload_test_results_tarball
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: ${{ steps.prepare_scenario.outputs.failure_summary_artifact_name }}
-          path: ${{ steps.prepare_scenario.outputs.failure_summary_artifact_name }}
+          name: ${{ steps.create_test_results_tarball.outputs.tarball_name }}
+          path: ${{ steps.create_test_results_tarball.outputs.tarball_path }}
+          retention-days: 7
           if-no-files-found: ignore
         continue-on-error: true