e2e-test.yaml

# Run e2e integration tests against the PR container image after PR Publish completes.
# Pulls the published PR image, runs the factory with test fixtures, and validates outputs.
name: E2E Tests

on:
  workflow_run:
    workflows: ["PR Publish"]
    types: [completed]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.workflow_run.id }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  REGISTRY: quay.io
  REGISTRY_IMAGE: rhdh-community/dynamic-plugins-factory

jobs:
  e2e-test:
    runs-on: ubuntu-latest
    if: github.event.workflow_run.conclusion == 'success'
    permissions:
      contents: read
    outputs:
      pr_number: ${{ steps.pr-info.outputs.pr_number }}
      image: ${{ steps.pr-info.outputs.image }}
      short_sha: ${{ steps.pr-info.outputs.short_sha }}
      pull_outcome: ${{ steps.pull-image.outcome }}
      test_outcome: ${{ steps.e2e-tests.outcome }}
    steps:
      - name: Download PR metadata
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: e2e-pr-metadata
          path: /tmp/e2e-pr-metadata
          run-id: ${{ github.event.workflow_run.id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Read PR metadata
        id: pr-info
        run: |
          if [[ ! -f "/tmp/e2e-pr-metadata/pr-info.json" ]]; then
            echo "Error: PR metadata file not found"
            exit 1
          fi

          PR_NUMBER=$(jq -r '.pr_number' /tmp/e2e-pr-metadata/pr-info.json)
          COMMIT_SHA=$(jq -r '.commit_sha' /tmp/e2e-pr-metadata/pr-info.json)
          SHORT_SHA=$(jq -r '.short_sha' /tmp/e2e-pr-metadata/pr-info.json)

          if ! [[ "$PR_NUMBER" =~ ^[0-9]{1,7}$ ]]; then
            echo "Error: Invalid PR number from metadata: '${PR_NUMBER}'"
            exit 1
          fi

          IMAGE="${REGISTRY}/${REGISTRY_IMAGE}:pr-${PR_NUMBER}"
          echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT
          echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
          echo "image=${IMAGE}" >> $GITHUB_OUTPUT
          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
          echo "Resolved PR #${PR_NUMBER}, commit: ${SHORT_SHA}, image: ${IMAGE}"

      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ steps.pr-info.outputs.commit_sha }}
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: '3.14'
          cache: 'pip'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install -r requirements.dev.txt

      - name: Pull container image
        id: pull-image
        continue-on-error: true
        env:
          IMAGE: ${{ steps.pr-info.outputs.image }}
        run: |
          echo "Pulling image: ${IMAGE}"
          if ! podman pull "${IMAGE}"; then
            echo "::warning::Failed to pull image '${IMAGE}'. The PR build may have been skipped or the image has expired."
            exit 1
          fi
          echo "Successfully pulled ${IMAGE}"
          podman inspect "${IMAGE}" --format '{{.Id}}'

      - name: Run E2E tests
        if: steps.pull-image.outcome == 'success'
        id: e2e-tests
        continue-on-error: true
        env:
          E2E_IMAGE: ${{ steps.pr-info.outputs.image }}
          E2E_CONTAINER_RUNTIME: podman
        run: |
          pytest tests/e2e/ -v --tb=long --junitxml=e2e-results.xml -m e2e -n auto --dist loadscope
      - name: Upload E2E test logs
        if: always() && steps.e2e-tests.outcome != 'skipped'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-test-logs-pr-${{ steps.pr-info.outputs.pr_number }}
          path: tests/e2e/logs/
          retention-days: 14
          if-no-files-found: ignore

      - name: Upload test results
        if: always() && steps.e2e-tests.outcome != 'skipped'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-test-results
          path: e2e-results.xml
          retention-days: 1
          if-no-files-found: ignore

  report-results:
    runs-on: ubuntu-latest
    needs: e2e-test
    if: always() && needs.e2e-test.result != 'skipped'
    permissions:
      pull-requests: write
    steps:
      - name: Download test results
        id: download-results
        continue-on-error: true
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: e2e-test-results

      - name: Generate test summary
        env:
          IMAGE: ${{ needs.e2e-test.outputs.image }}
          PR_NUMBER: ${{ needs.e2e-test.outputs.pr_number }}
          PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }}
          E2E_RUN_ID: ${{ github.run_id }}
          SERVER_URL: ${{ github.server_url }}
          REPOSITORY: ${{ github.repository }}
          PULL_OUTCOME: ${{ needs.e2e-test.outputs.pull_outcome }}
        run: |
          python3 << 'PYTHON_SCRIPT'
          import xml.etree.ElementTree as ET
          import os

          image = os.environ["IMAGE"]
          pr_number = os.environ["PR_NUMBER"]
          publish_run_id = os.environ["PUBLISH_RUN_ID"]
          e2e_run_id = os.environ["E2E_RUN_ID"]
          server_url = os.environ["SERVER_URL"]
          repository = os.environ["REPOSITORY"]
          pull_outcome = os.environ["PULL_OUTCOME"]
          summary_file = os.environ.get("GITHUB_STEP_SUMMARY", "/dev/stdout")

          with open(summary_file, "a") as f:
              if pull_outcome != "success":
                  f.write("## ⚠️ E2E Tests — Image Not Available\n\n")
                  f.write(f"**Image:** `{image}`\n")
                  f.write(f"**PR:** #{pr_number}\n\n")
                  f.write("The PR container image could not be pulled. "
                          "This typically means the PR build was skipped "
                          "(e.g., only non-build files were changed) or the image has expired.\n\n")
                  f.write("E2E tests were **not executed**.\n\n")
              else:
                  try:
                      tree = ET.parse("e2e-results.xml")
                      root = tree.getroot()
                      suite = root.find("testsuite") if root.tag != "testsuite" else root

                      tests = int(suite.get("tests", "0"))
                      failures = int(suite.get("failures", "0"))
                      errors = int(suite.get("errors", "0"))
                      skipped = int(suite.get("skipped", "0"))
                      time_taken = float(suite.get("time", "0"))
                      passed = tests - failures - errors - skipped

                      all_passed = (failures + errors) == 0
                      status_emoji = "✅" if all_passed else "❌"
                      status_text = "Passed" if all_passed else "Failed"

                      f.write(f"## {status_emoji} E2E Test Results — {status_text}\n\n")
                      f.write(f"**Image:** `{image}`\n")
                      f.write(f"**PR:** #{pr_number}\n")
                      f.write(f"**Duration:** {time_taken:.1f}s\n\n")

                      f.write("| Status | Count |\n")
                      f.write("|--------|-------|\n")
                      f.write(f"| ✅ Passed | {passed} |\n")
                      f.write(f"| ❌ Failed | {failures} |\n")
                      f.write(f"| 💥 Errors | {errors} |\n")
                      f.write(f"| ⏭️ Skipped | {skipped} |\n")
                      f.write(f"| **Total** | **{tests}** |\n\n")

                      f.write("### Test Details\n\n")
                      f.write("| Test | Status | Duration |\n")
                      f.write("|------|--------|----------|\n")

                      for tc in suite.iter("testcase"):
                          name = tc.get("name", "unknown")
                          tc_time = float(tc.get("time", "0"))

                          failure = tc.find("failure")
                          error = tc.find("error")
                          skip = tc.find("skipped")

                          if failure is not None:
                              tc_status = "❌ Failed"
                          elif error is not None:
                              tc_status = "💥 Error"
                          elif skip is not None:
                              tc_status = "⏭️ Skipped"
                          else:
                              tc_status = "✅ Passed"

                          f.write(f"| `{name}` | {tc_status} | {tc_time:.1f}s |\n")

                      f.write("\n")

                      has_failures = False
                      for tc in suite.iter("testcase"):
                          failure = tc.find("failure")
                          error = tc.find("error")
                          detail = failure if failure is not None else error

                          if detail is not None:
                              if not has_failures:
                                  f.write("### Failure Details\n\n")
                                  has_failures = True

                              name = tc.get("name", "unknown")
                              message = detail.get("message", "")
                              text = detail.text or ""
                              if len(text) > 3000:
                                  text = text[:3000] + "\n... (truncated)"

                              f.write(f"<details>\n<summary><code>{name}</code>: {message[:200]}</summary>\n\n")
                              f.write(f"```\n{text}\n```\n\n")
                              f.write("</details>\n\n")

                  except FileNotFoundError:
                      f.write("## ❌ E2E Test Results\n\n")
                      f.write(f"**Image:** `{image}`\n")
                      f.write(f"**PR:** #{pr_number}\n\n")
                      f.write("No test results file found. Tests may have failed to start.\n\n")

              f.write("### Traceability\n\n")
              f.write(f"- **Publish:** [{server_url}/{repository}/actions/runs/{publish_run_id}]")
              f.write(f"({server_url}/{repository}/actions/runs/{publish_run_id})\n")
              f.write(f"- **E2E:** [{server_url}/{repository}/actions/runs/{e2e_run_id}]")
              f.write(f"({server_url}/{repository}/actions/runs/{e2e_run_id})\n")
          PYTHON_SCRIPT

      - name: Comment on PR
        if: always()
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          PR_NUMBER: ${{ needs.e2e-test.outputs.pr_number }}
          IMAGE: ${{ needs.e2e-test.outputs.image }}
          E2E_RUN_ID: ${{ github.run_id }}
          PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }}
          TEST_OUTCOME: ${{ needs.e2e-test.outputs.test_outcome }}
          PULL_OUTCOME: ${{ needs.e2e-test.outputs.pull_outcome }}
        with:
          script: |
            const prNumber = parseInt(process.env.PR_NUMBER);
            const image = process.env.IMAGE;
            const e2eRunId = process.env.E2E_RUN_ID;
            const publishRunId = process.env.PUBLISH_RUN_ID;
            const testOutcome = process.env.TEST_OUTCOME;
            const pullOutcome = process.env.PULL_OUTCOME;

            const e2eUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${e2eRunId}`;
            const publishUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${publishRunId}`;

            let statusEmoji, statusText, statusDetail, testSummary;

            if (pullOutcome !== 'success') {
              statusEmoji = '⚠️';
              statusText = 'Skipped — Image Not Available';
              statusDetail = [
                'The PR container image could not be pulled. This typically means the PR build',
                'was skipped (e.g., only non-build files were changed) or the image has expired.',
                '',
                'E2E tests were **not executed**.',
              ].join('\n');
              testSummary = '';
            } else {
              statusEmoji = testOutcome === 'success' ? '✅' : '❌';
              statusText = testOutcome === 'success' ? 'Passed' : 'Failed';
              statusDetail = testOutcome === 'success'
                ? 'All end-to-end integration tests passed for the container image.'
                : 'End-to-end integration tests failed for the container image. See the workflow run for details.';

              testSummary = '';
              const fs = require('fs');
              try {
                const xml = fs.readFileSync('e2e-results.xml', 'utf8');
                const testsMatch = xml.match(/tests="(\d+)"/);
                const failuresMatch = xml.match(/failures="(\d+)"/);
                const errorsMatch = xml.match(/errors="(\d+)"/);
                const timeMatch = xml.match(/time="([\d.]+)"/);

                const total = testsMatch ? parseInt(testsMatch[1]) : 0;
                const failures = failuresMatch ? parseInt(failuresMatch[1]) : 0;
                const errors = errorsMatch ? parseInt(errorsMatch[1]) : 0;
                const time = timeMatch ? parseFloat(timeMatch[1]).toFixed(1) : '?';
                const passed = total - failures - errors;

                testSummary = [
                  '| Tests | Passed | Failed | Duration |',
                  '|-------|--------|--------|----------|',
                  `| ${total} | ${passed} | ${failures + errors} | ${time}s |`,
                ].join('\n');
              } catch (e) {
                testSummary = '_No detailed results available._';
              }
            }

            const bodyParts = [
              `## ${statusEmoji} E2E Tests ${statusText}`,
              '',
              statusDetail,
              '',
              `**Image:** \`${image}\``,
            ];

            if (testSummary) {
              bodyParts.push('', testSummary);
            }

            bodyParts.push(
              '',
              '### Traceability',
              '',
              `- **Publish:** [PR Publish #${publishRunId}](${publishUrl})`,
              `- **E2E:** [E2E Tests #${e2eRunId}](${e2eUrl})`,
            );

            const body = bodyParts.join('\n');

            await github.rest.issues.createComment({
              ...context.repo,
              issue_number: prNumber,
              body: body,
            });

      - name: Fail workflow if tests failed
        if: always() && needs.e2e-test.outputs.test_outcome == 'failure'
        run: |
          echo "E2E tests failed. Failing workflow."
          exit 1

  # Handle failed upstream workflow (PR Publish failed)
  notify-publish-failure:
    runs-on: ubuntu-latest
    if: github.event.workflow_run.conclusion == 'failure'
    permissions:
      actions: read
      pull-requests: write
    steps:
      - name: Download PR metadata
        id: download-metadata
        continue-on-error: true
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: e2e-pr-metadata
          path: /tmp/e2e-pr-metadata
          run-id: ${{ github.event.workflow_run.id }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Read PR metadata
        id: pr-info
        if: steps.download-metadata.outcome == 'success'
        run: |
          PR_NUMBER=$(jq -r '.pr_number' /tmp/e2e-pr-metadata/pr-info.json)
          if [[ "$PR_NUMBER" =~ ^[0-9]{1,7}$ ]]; then
            echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT
          fi

      - name: E2E Skipped Summary
        env:
          PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }}
          SERVER_URL: ${{ github.server_url }}
          REPOSITORY: ${{ github.repository }}
        run: |
          echo "## ⏭️ E2E Tests Skipped" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "The upstream PR Publish workflow failed. E2E tests were not run." >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- **PR Publish:** [Run #${PUBLISH_RUN_ID}](${SERVER_URL}/${REPOSITORY}/actions/runs/${PUBLISH_RUN_ID}) (failed)" >> $GITHUB_STEP_SUMMARY

      - name: Comment on PR
        if: steps.pr-info.outputs.pr_number != ''
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          PR_NUMBER: ${{ steps.pr-info.outputs.pr_number }}
          E2E_RUN_ID: ${{ github.run_id }}
          PUBLISH_RUN_ID: ${{ github.event.workflow_run.id }}
        with:
          script: |
            const prNumber = parseInt(process.env.PR_NUMBER);
            const e2eRunId = process.env.E2E_RUN_ID;
            const publishRunId = process.env.PUBLISH_RUN_ID;

            const e2eUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${e2eRunId}`;
            const publishUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${publishRunId}`;

            const body = [
              '## ⏭️ E2E Tests Skipped',
              '',
              'The upstream PR Publish workflow failed. E2E tests were **not executed**.',
              '',
              '### Traceability',
              '',
              `- **Publish:** [PR Publish #${publishRunId}](${publishUrl}) (failed)`,
              `- **E2E:** [E2E Tests #${e2eRunId}](${e2eUrl})`,
            ].join('\n');

            await github.rest.issues.createComment({
              ...context.repo,
              issue_number: prNumber,
              body: body,
            });