diff --git a/.github/workflows/pr-actions.yaml b/.github/workflows/pr-actions.yaml index ef7311350..e4220a790 100644 --- a/.github/workflows/pr-actions.yaml +++ b/.github/workflows/pr-actions.yaml @@ -12,6 +12,10 @@ on: description: Command to execute (publish, update-versions, update-commit, smoketest) type: string required: true + smoketest-tag: + description: Optional RHDH tag override for `/smoketest ` + type: string + required: false jobs: parse: @@ -31,12 +35,16 @@ jobs: command-name: ${{ steps.extract.outputs.command-name }} error-message: ${{ steps.extract.outputs.error-message }} pr-number: ${{ steps.extract.outputs.pr-number }} + smoketest-tag: ${{ steps.extract.outputs.smoketest-tag }} steps: - name: Extract command from comment id: extract uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | + const SMOKETEST_TAG_RE = /^(?:pr-[0-9]+(?:-[0-9a-f]{7,40})?|next(?:-[0-9]+\.[0-9]+-[0-9a-f]{7,40}|-[0-9a-f]{7,40})?)$/; + const SMOKETEST_HELP = 'Use /smoketest or /smoketest . Allowed tags: pr-, pr--, next, next-.-, next-.'; + core.setOutput('smoketest-tag', ''); if (context.eventName === 'workflow_dispatch') { const commandName = '${{ inputs.command-name }}'; const allowed = new Set(['publish', 'update-versions', 'update-commit', 'smoketest']); @@ -50,6 +58,24 @@ jobs: core.setOutput('command-name', commandName); core.setOutput('error-message', ''); core.setOutput('pr-number', '${{ inputs.pr-number }}'); + const smoketestTag = String('${{ inputs.smoketest-tag }}').trim(); + if (smoketestTag !== '' && commandName !== 'smoketest') { + const errorMsg = 'Input smoketest-tag is only valid with command-name=smoketest.'; + core.setOutput('error-message', errorMsg); + core.setOutput('command-name', ''); + core.setFailed(errorMsg); + return; + } + if (commandName === 'smoketest') { + if (smoketestTag !== '' && !SMOKETEST_TAG_RE.test(smoketestTag)) { + const errorMsg = `Invalid smoketest tag: ${smoketestTag}. ${SMOKETEST_HELP}`; + core.setOutput('error-message', errorMsg); + core.setOutput('command-name', ''); + core.setFailed(errorMsg); + return; + } + core.setOutput('smoketest-tag', smoketestTag); + } return; } core.setOutput('pr-number', String(context.issue.number)); @@ -58,8 +84,37 @@ jobs: .split(/\r?\n/) .map(l => l.trim()) .filter(l => l.length > 0); - const allowed = new Set(['/publish', '/update-versions', '/update-commit', '/smoketest']); - const matchingCommands = lines.filter(l => allowed.has(l)); + const allowed = new Set(['/publish', '/update-versions', '/update-commit']); + const matchingCommands = []; + let smoketestTag = ''; + let malformedSmoketestCommand = ''; + for (const line of lines) { + if (allowed.has(line)) { + matchingCommands.push(line); + continue; + } + if (line.startsWith('/smoketest')) { + const smoketestMatch = line.match(/^\/smoketest(?:\s+(\S+))?$/); + if (!smoketestMatch) { + malformedSmoketestCommand = line; + break; + } + const parsedTag = smoketestMatch[1] ?? ''; + if (parsedTag !== '' && !SMOKETEST_TAG_RE.test(parsedTag)) { + malformedSmoketestCommand = line; + break; + } + matchingCommands.push('/smoketest'); + smoketestTag = parsedTag; + } + } + if (malformedSmoketestCommand) { + const errorMsg = `Invalid smoketest command: "${malformedSmoketestCommand}". ${SMOKETEST_HELP}`; + core.setOutput('error-message', errorMsg); + core.setOutput('command-name', ''); + core.setFailed(errorMsg); + return; + } if (matchingCommands.length > 1) { const errorMsg = `Multiple commands found in comment: ${matchingCommands.join(', ')}. Please use only one command per comment.`; @@ -80,6 +135,7 @@ jobs: const firstMatching = matchingCommands[0] || ''; core.setOutput('command-name', firstMatching.startsWith('/') ? firstMatching.slice(1) : firstMatching); + core.setOutput('smoketest-tag', firstMatching === '/smoketest' ? smoketestTag : ''); core.setOutput('error-message', ''); add_error_comment: @@ -105,7 +161,7 @@ jobs: script: | const errorMessage = core.getInput('error_message'); const prNumber = Number(core.getInput('pr_number')); - const body = `**Error**: ${errorMessage}\n\nValid commands are:\n- \`/publish\` - Publish dynamic plugin images\n- \`/update-versions\` - Update versions from release branch\n- \`/update-commit\` - Update commit from automatic discovery\n- \`/smoketest\` - Run smoke tests`; + const body = `**Error**: ${errorMessage}\n\nValid commands are:\n- \`/publish\` - Publish dynamic plugin images\n- \`/update-versions\` - Update versions from release branch\n- \`/update-commit\` - Update commit from automatic discovery\n- \`/smoketest\` - Run smoke tests with default image\n- \`/smoketest \` - Run smoke tests with \`quay.io/rhdh-community/rhdh:\`\n - Allowed tags: \`pr-4907\`, \`pr-4929-90eff067\`, \`next\`, \`next-1.10-244a2755\`, \`next-8a0d43e7\``; await github.rest.issues.createComment({ issue_number: prNumber, owner: context.repo.owner, @@ -131,6 +187,7 @@ jobs: overlay-commit: ${{ steps.get-branch.outputs.overlay-commit }} workspace: ${{ steps.get-branch.outputs.workspace }} pr-number: ${{ steps.get-branch.outputs.pr-number }} + rhdh-tag: ${{ needs.parse.outputs.smoketest-tag }} permissions: statuses: write @@ -254,6 +311,7 @@ jobs: 'overlay-commit': '${{ needs.prepare.outputs.overlay-commit }}', 'pr-number': String(${{ needs.prepare.outputs.pr-number }}), 'target-branch': '${{ needs.prepare.outputs.target-branch }}', + 'rhdh-tag': '${{ needs.prepare.outputs.rhdh-tag }}', }, }); diff --git a/.github/workflows/run-workspace-smoke-tests.yaml b/.github/workflows/run-workspace-smoke-tests.yaml index b52e81333..41ae9e47f 100644 --- a/.github/workflows/run-workspace-smoke-tests.yaml +++ b/.github/workflows/run-workspace-smoke-tests.yaml @@ -7,6 +7,10 @@ on: description: PR target branch (main or release-X.Y) type: string default: main + rhdh-tag: + description: Optional RHDH image tag override for quay.io/rhdh-community/rhdh: + type: string + required: false outputs: success: description: Overall smoke test result @@ -17,6 +21,9 @@ on: error-logs: description: Extracted error messages from container logs value: ${{ jobs.run.outputs.error-logs }} + image-ref: + description: Resolved RHDH container image used for smoke tests + value: ${{ jobs.run.outputs.image-ref }} jobs: run: @@ -29,6 +36,7 @@ jobs: success: ${{ steps.collect-results.outputs.success }} failed-plugins: ${{ steps.collect-results.outputs.failed-plugins }} error-logs: ${{ steps.capture-errors.outputs.error-logs }} + image-ref: ${{ steps.start-rhdh.outputs.image-ref }} steps: - name: Download smoke test artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -44,6 +52,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Start RHDH with test plugins config + id: start-rhdh + env: + INPUT_RHDH_TAG: ${{ inputs.rhdh-tag }} + INPUT_TARGET_BRANCH: ${{ inputs.target-branch }} run: | set -euo pipefail ls -la ./artifacts/ || true @@ -72,17 +84,30 @@ jobs: DOCKER_CMD="$DOCKER_CMD -v $HOME/.docker/config.json:/root/.docker/config.json:ro" DOCKER_CMD="$DOCKER_CMD -e REGISTRY_AUTH_FILE=/root/.docker/config.json" - # Derive image tag from target branch - TARGET_BRANCH="${{ inputs.target-branch }}" - if [[ "$TARGET_BRANCH" =~ ^release-([0-9]+\.[0-9]+)$ ]]; then - IMAGE_TAG="next-${BASH_REMATCH[1]}" + # Resolve image from explicit /smoketest override or target branch default + RHDH_TAG="${INPUT_RHDH_TAG:-}" + if [[ -n "$RHDH_TAG" ]]; then + if [[ ! "$RHDH_TAG" =~ ^(pr-[0-9]+(-[0-9a-f]{7,40})?|next(-[0-9]+\.[0-9]+-[0-9a-f]{7,40}|-[0-9a-f]{7,40})?)$ ]]; then + echo "Invalid rhdh-tag input: $RHDH_TAG." + echo "Allowed tags: pr-, pr--, next, next-.-, next-." + exit 1 + fi + IMAGE_REF="quay.io/rhdh-community/rhdh:${RHDH_TAG}" + echo "Using explicit RHDH image override from smoketest tag: $IMAGE_REF" else - IMAGE_TAG="next" + TARGET_BRANCH="${INPUT_TARGET_BRANCH:-main}" + if [[ "$TARGET_BRANCH" =~ ^release-([0-9]+\.[0-9]+)$ ]]; then + IMAGE_TAG="next-${BASH_REMATCH[1]}" + else + IMAGE_TAG="next" + fi + IMAGE_REF="quay.io/rhdh-community/rhdh:${IMAGE_TAG}" + echo "Using default RHDH image tag: $IMAGE_TAG (target branch: $TARGET_BRANCH)" fi - echo "Using RHDH image tag: $IMAGE_TAG (target branch: $TARGET_BRANCH)" + echo "image-ref=$IMAGE_REF" >> "$GITHUB_OUTPUT" # Add image and command - DOCKER_CMD="$DOCKER_CMD --entrypoint /bin/bash quay.io/rhdh-community/rhdh:${IMAGE_TAG} -c '" + DOCKER_CMD="$DOCKER_CMD --entrypoint /bin/bash ${IMAGE_REF} -c '" DOCKER_CMD="$DOCKER_CMD set -ex; " DOCKER_CMD="$DOCKER_CMD PLUGINS_ROOT=/opt/app-root/src/dynamic-plugins-root; " DOCKER_CMD="$DOCKER_CMD GENERATED_CONFIG=\$PLUGINS_ROOT/app-config.dynamic-plugins.yaml; " diff --git a/.github/workflows/workspace-tests.yaml b/.github/workflows/workspace-tests.yaml index 907e81788..ae5f711b5 100644 --- a/.github/workflows/workspace-tests.yaml +++ b/.github/workflows/workspace-tests.yaml @@ -21,6 +21,9 @@ on: target-branch: type: string required: true + rhdh-tag: + type: string + required: false jobs: resolve: @@ -37,6 +40,7 @@ jobs: pr-number: ${{ inputs.pr-number }} published-exports: ${{ steps.meta.outputs.published-exports }} target-branch: ${{ inputs.target-branch }} + rhdh-tag: ${{ inputs.rhdh-tag }} steps: - name: Download published-exports artifact for this PR if: inputs.workspace != '' && inputs.pr-number != '' @@ -306,6 +310,7 @@ jobs: uses: ./.github/workflows/run-workspace-smoke-tests.yaml with: target-branch: ${{ needs.resolve.outputs.target-branch }} + rhdh-tag: ${{ needs.resolve.outputs.rhdh-tag }} add-skipped-test-comment: if: ${{ always() && needs.resolve.outputs.pr-number != 'null' && needs.resolve.outputs.pr-number != '' && (needs.resolve.outputs.workspace == '' || (needs.prepare-test-config.result != 'skipped' && (needs.prepare-test-config.outputs.has-runnable-plugins != 'true' || needs.prepare-test-config.outputs.skip-tests-missing-env == 'true'))) }} diff --git a/README.md b/README.md index 9e9ebc302..1f79cb588 100644 --- a/README.md +++ b/README.md @@ -135,8 +135,11 @@ The repository includes an automated smoke testing workflow that verifies plugin **Triggering smoke tests:** - After `/publish`: Smoke tests run automatically upon successful publish completion -- Manual testing: Use `/smoketest` comment on the PR to rerun the smoke tests using the latest published artifacts - - For `/smoketest` command, a previous successful `/publish` run is required +- Manual testing: Use one of these comments on the PR to rerun smoke tests using the latest published artifacts: + - `/smoketest` (default image derived from PR target branch) + - `/smoketest ` (uses `quay.io/rhdh-community/rhdh:`) + - Allowed tags: `pr-4907`, `pr-4929-90eff067`, `next`, `next-1.10-244a2755`, `next-8a0d43e7` + - A previous successful `/publish` run is required **Smoke testing workflow steps:** 1. **Resolve metadata**: Retrieves published OCI references and PR metadata from the `published-exports` artifact diff --git a/user-guide/01-getting-started.md b/user-guide/01-getting-started.md index ad1517dd7..33bbbdf48 100644 --- a/user-guide/01-getting-started.md +++ b/user-guide/01-getting-started.md @@ -288,6 +288,15 @@ To re-run smoke tests manually: /smoketest ``` +Use this to override the RHDH container image with a PR tag from `quay.io/rhdh-community/rhdh`: + +``` +/smoketest pr-4929-90eff067 +``` + +`/smoketest ` resolves to `quay.io/rhdh-community/rhdh:`. +Allowed tags include `pr-4907`, `pr-4929-90eff067`, `next`, `next-1.10-244a2755`, and `next-8a0d43e7`. + Plugin-specific configuration is extracted from `spec.appConfigExamples[0].content` in each plugin's metadata file and placed under `pluginConfig` in the generated config. The optional workspace-level `app-config.test.yaml` is for test-only or shared workspace settings. If a plugin's config references environment variables (e.g., `${API_TOKEN}`), provide them in `workspaces//smoke-tests/test.env`. ### Manual Testing diff --git a/user-guide/02-export-tools.md b/user-guide/02-export-tools.md index ed64a294a..c3f360481 100644 --- a/user-guide/02-export-tools.md +++ b/user-guide/02-export-tools.md @@ -118,7 +118,8 @@ When working with Pull Requests, use these comment commands: | Command | Action | |---------|--------| | `/publish` | Build and publish test OCI artifacts | -| `/smoketest` | Re-run smoke tests (requires prior `/publish`) | +| `/smoketest` | Re-run smoke tests with the default branch-derived RHDH image (requires prior `/publish`) | +| `/smoketest ` | Re-run smoke tests with `quay.io/rhdh-community/rhdh:` (allowlisted tag formats, requires prior `/publish`) | ### What `/publish` Does diff --git a/user-guide/05-version-updates.md b/user-guide/05-version-updates.md index f94a5d795..e55d9a80a 100644 --- a/user-guide/05-version-updates.md +++ b/user-guide/05-version-updates.md @@ -147,8 +147,12 @@ git push origin update-your-plugin-version # Open PR, then comment: # /publish # /smoketest +# or: /smoketest pr-4929-90eff067 ``` +`/smoketest ` resolves to `quay.io/rhdh-community/rhdh:` for the smoke-test run. +Allowed tags include `pr-4907`, `pr-4929-90eff067`, `next`, `next-1.10-244a2755`, and `next-8a0d43e7`. + --- ## Handling Breaking Backstage Updates