Add CLI E2E Recording Comment #12610
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Add CLI E2E Recording Comment | |
| on: | |
| # Trigger when the CI workflow completes (success, failure, or cancelled) | |
| # We want to post recordings even if CI was cancelled since recordings may still exist | |
| workflow_run: | |
| workflows: ["CI"] | |
| types: | |
| - completed | |
| # Allow manual triggering for testing | |
| workflow_dispatch: | |
| inputs: | |
| run_id: | |
| description: 'Workflow run ID to download artifacts from' | |
| required: true | |
| type: number | |
| pr_number: | |
| description: 'Optional PR number to comment on (skips the head-SHA lookup; useful for testing against merged PRs)' | |
| required: false | |
| type: number | |
| jobs: | |
| add-recording-comment: | |
| # Only run on the dotnet org and for pull requests | |
| # Note: This runs for all conclusions (success, failure, cancelled) since recordings may exist | |
| if: >- | |
| ${{ github.repository_owner == 'microsoft' && | |
| (github.event.workflow_run.event == 'pull_request' || github.event_name == 'workflow_dispatch') }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| actions: read | |
| steps: | |
| - name: Log workflow run info | |
| if: ${{ github.event_name == 'workflow_run' }} | |
| run: | | |
| echo "CI workflow conclusion: ${{ github.event.workflow_run.conclusion }}" | |
| echo "CI workflow run ID: ${{ github.event.workflow_run.id }}" | |
| - name: Get workflow run info | |
| id: run-info | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| let runId, prNumber, headSha; | |
| if (context.eventName === 'workflow_dispatch') { | |
| // Manual trigger - get run info from input | |
| runId = context.payload.inputs.run_id; | |
| const run = await github.rest.actions.getWorkflowRun({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: runId | |
| }); | |
| headSha = run.data.head_sha; | |
| // Allow explicit PR override so we can dry-run against a merged PR's | |
| // artifacts without depending on the open-PR head-SHA lookup below. | |
| const overridePr = context.payload.inputs.pr_number; | |
| if (overridePr) { | |
| prNumber = Number(overridePr); | |
| } else { | |
| // Find PR by head SHA | |
| const prs = await github.rest.pulls.list({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| head: `${context.repo.owner}:${run.data.head_branch}` | |
| }); | |
| prNumber = prs.data.length > 0 ? prs.data[0].number : null; | |
| } | |
| } else { | |
| // Triggered by workflow_run | |
| runId = context.payload.workflow_run.id; | |
| headSha = context.payload.workflow_run.head_sha; | |
| // Get PR number from the workflow run | |
| const prs = context.payload.workflow_run.pull_requests; | |
| prNumber = prs && prs.length > 0 ? prs[0].number : null; | |
| } | |
| if (!prNumber) { | |
| console.log('No PR found for this workflow run, skipping comment'); | |
| core.setOutput('skip', 'true'); | |
| return; | |
| } | |
| core.setOutput('skip', 'false'); | |
| core.setOutput('run_id', runId); | |
| core.setOutput('pr_number', prNumber); | |
| core.setOutput('head_sha', headSha); | |
| console.log(`Run ID: ${runId}, PR: ${prNumber}, SHA: ${headSha}`); | |
| - name: Download CLI E2E test artifacts | |
| if: steps.run-info.outputs.skip != 'true' | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const runId = ${{ steps.run-info.outputs.run_id }}; | |
| // List ALL artifacts for the workflow run using pagination | |
| // (without pagination we only get the first page and miss CLI E2E artifacts) | |
| const allArtifacts = await github.paginate( | |
| github.rest.actions.listWorkflowRunArtifacts, | |
| { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: runId, | |
| per_page: 100 | |
| } | |
| ); | |
| console.log(`Total artifacts found: ${allArtifacts.length}`); | |
| // Filter for CLI E2E recording artifacts (simple pattern match) | |
| // These are uploaded by the run-tests workflow with name: cli-e2e-recordings-{TestName} | |
| const cliE2eArtifacts = allArtifacts.filter(a => | |
| a.name.startsWith('cli-e2e-recordings-') | |
| ); | |
| console.log(`Found ${cliE2eArtifacts.length} CLI E2E recording artifacts`); | |
| // Create recordings directory | |
| const recordingsDir = 'recordings'; | |
| fs.mkdirSync(recordingsDir, { recursive: true }); | |
| // Download each artifact | |
| for (const artifact of cliE2eArtifacts) { | |
| console.log(`Downloading ${artifact.name}...`); | |
| const download = await github.rest.actions.downloadArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: artifact.id, | |
| archive_format: 'zip' | |
| }); | |
| const artifactPath = path.join(recordingsDir, `${artifact.name}.zip`); | |
| fs.writeFileSync(artifactPath, Buffer.from(download.data)); | |
| console.log(`Saved to ${artifactPath}`); | |
| } | |
| core.setOutput('artifact_count', cliE2eArtifacts.length); | |
| - name: Extract recordings from artifacts | |
| if: steps.run-info.outputs.skip != 'true' | |
| shell: bash | |
| run: | | |
| mkdir -p cast_files | |
| mkdir -p trx_files | |
| for zipfile in recordings/*.zip; do | |
| if [ -f "$zipfile" ]; then | |
| echo "Extracting $zipfile..." | |
| unzip -o "$zipfile" -d "recordings/extracted_$(basename "$zipfile" .zip)" || true | |
| fi | |
| done | |
| # Find and copy all .cast files | |
| find recordings -name "*.cast" -exec cp {} cast_files/ \; 2>/dev/null || true | |
| # Find and copy all .trx files (test results) | |
| find recordings -name "*.trx" -exec cp {} trx_files/ \; 2>/dev/null || true | |
| echo "Found recordings:" | |
| ls -la cast_files/ || echo "No .cast files found" | |
| echo "Found TRX files:" | |
| ls -la trx_files/ || echo "No .trx files found" | |
| - name: Parse test results from TRX files | |
| if: steps.run-info.outputs.skip != 'true' && hashFiles('cast_files/*.cast') != '' | |
| id: test-results | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| # Parse TRX (XML) files to extract test method outcomes using yq + jq | |
| # (both pre-installed on ubuntu-latest). Produces a JSON map of | |
| # testMethodName -> outcome for the bash comment step to consume. | |
| # When the same test appears in multiple TRX files (e.g. retries), | |
| # "Failed" wins over other outcomes. | |
| if compgen -G trx_files/*.trx > /dev/null 2>&1; then | |
| echo "Parsing TRX files with yq..." | |
| # Convert each TRX to JSON with yq (Go yq's expression language is | |
| # limited, so do all reshaping in jq). The resulting documents look like: | |
| # { "TestRun": { "Results": { "UnitTestResult": <object|array> } } } | |
| # Attribute keys are exposed by yq as "+@<attr>" (newer yq) or | |
| # "@<attr>" (older yq), and a single result is emitted as an object | |
| # rather than an array, so jq must handle both shapes. | |
| yq -p xml -o json '.' trx_files/*.trx \ | |
| | jq -s ' | |
| def attr(o; k): o["+@" + k] // o["@" + k]; | |
| # Best-effort: also key by the bare method name (with theory | |
| # parameter data stripped) so a .cast file named after the | |
| # CallerMemberName matches a TRX entry like | |
| # "Namespace.Class.Method(toolchain: "pnpm")". | |
| def bare_method(name): | |
| (name | split(".") | last) | sub("\\(.*$"; ""); | |
| def fqn_no_params(name): | |
| name | sub("\\(.*$"; ""); | |
| def merge(map; key; outcome): | |
| if map[key] == "Failed" then map else map + {(key): outcome} end; | |
| reduce ( | |
| .[] | |
| | .TestRun.Results.UnitTestResult | |
| | (if type == "array" then .[] else . end) | |
| ) as $r ({}; | |
| attr($r; "testName") as $name | | |
| attr($r; "outcome") as $outcome | | |
| if $name == null or $outcome == null then . | |
| else | |
| merge(.; bare_method($name); $outcome) | |
| | merge(.; fqn_no_params($name); $outcome) | |
| | merge(.; $name; $outcome) | |
| end | |
| ) | |
| ' > test_outcomes.json | |
| OUTCOME_COUNT=$(jq 'length' test_outcomes.json) | |
| echo "Parsed $OUTCOME_COUNT test outcome(s)" | |
| if [ "$OUTCOME_COUNT" -gt 0 ]; then | |
| echo "has_outcomes=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_outcomes=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| echo "No TRX files found" | |
| echo '{}' > test_outcomes.json | |
| echo "has_outcomes=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Upload recordings and post comment | |
| if: steps.run-info.outputs.skip != 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} | |
| GITHUB_EVENT_REPO_NAME: ${{ github.event.repository.name }} | |
| shell: bash | |
| run: | | |
| PR_NUMBER="${{ steps.run-info.outputs.pr_number }}" | |
| RUN_ID="${{ steps.run-info.outputs.run_id }}" | |
| HEAD_SHA="${{ steps.run-info.outputs.head_sha }}" | |
| SHORT_SHA="${HEAD_SHA:0:7}" | |
| RECORDINGS_DIR="cast_files" | |
| if [ -d "$RECORDINGS_DIR" ] && compgen -G "$RECORDINGS_DIR"/*.cast > /dev/null; then | |
| # Install asciinema | |
| pip install asciinema | |
| # Load test outcomes from TRX parsing step (JSON: {"methodName": "Passed|Failed", ...}) | |
| HAS_OUTCOMES="${{ steps.test-results.outputs.has_outcomes }}" | |
| if [ "$HAS_OUTCOMES" = "true" ] && [ -f "test_outcomes.json" ]; then | |
| echo "Loaded test outcomes from TRX files" | |
| else | |
| echo "No test outcomes available, will show recordings without pass/fail status" | |
| echo '{}' > test_outcomes.json | |
| fi | |
| # Unique marker to identify CLI E2E recording comments | |
| COMMENT_MARKER="<!-- cli-e2e-recordings -->" | |
| # Retry configuration for asciinema uploads | |
| MAX_UPLOAD_RETRIES=5 | |
| RETRY_BASE_DELAY_SECONDS=30 | |
| UPLOAD_COUNT=0 | |
| FAIL_COUNT=0 | |
| TOTAL_COUNT=0 | |
| TEST_PASS_COUNT=0 | |
| TEST_FAIL_COUNT=0 | |
| TEST_UNKNOWN_COUNT=0 | |
| # Arrays to track failed test recordings separately | |
| FAILED_TESTS_BODY="" | |
| TABLE_BODY="" | |
| for castfile in "$RECORDINGS_DIR"/*.cast; do | |
| if [ -f "$castfile" ]; then | |
| filename=$(basename "$castfile" .cast) | |
| echo "Uploading $castfile..." | |
| TOTAL_COUNT=$((TOTAL_COUNT + 1)) | |
| # Sanitize filename for safe markdown rendering. | |
| # .cast files are named via [CallerMemberName] so should be valid C# identifiers, | |
| # but we sanitize defensively since this runs in a privileged workflow_run context | |
| # and artifacts could come from fork PRs. | |
| safe_filename=$(echo "$filename" | tr -cd 'A-Za-z0-9_.-') | |
| # Look up test outcome from TRX data. | |
| # .cast files are named after the test method name (via [CallerMemberName] in CreateTestTerminal), | |
| # so the filename matches the method name key in the outcomes JSON. | |
| # Per-link label carries the outcome too, so a recording URL copied out of | |
| # the table still tells reviewers whether it represents a pass or a failure. | |
| TEST_OUTCOME=$(jq -r --arg name "$filename" '.[$name] // "Unknown"' test_outcomes.json) | |
| if [ "$TEST_OUTCOME" = "Passed" ]; then | |
| STATUS_EMOJI="✅" | |
| LINK_LABEL="✅ ▶️ View recording" | |
| TEST_PASS_COUNT=$((TEST_PASS_COUNT + 1)) | |
| elif [ "$TEST_OUTCOME" = "Failed" ]; then | |
| STATUS_EMOJI="❌" | |
| LINK_LABEL="❌ ▶️ View failure recording" | |
| TEST_FAIL_COUNT=$((TEST_FAIL_COUNT + 1)) | |
| else | |
| STATUS_EMOJI="❔" | |
| LINK_LABEL="❔ ▶️ View recording" | |
| TEST_UNKNOWN_COUNT=$((TEST_UNKNOWN_COUNT + 1)) | |
| fi | |
| # Upload to asciinema with retry logic for transient failures | |
| ASCIINEMA_URL="" | |
| for attempt in $(seq 1 "$MAX_UPLOAD_RETRIES"); do | |
| UPLOAD_OUTPUT=$(asciinema upload "$castfile" 2>&1) || true | |
| ASCIINEMA_URL=$(echo "$UPLOAD_OUTPUT" | grep -oP 'https://asciinema\.org/a/[a-zA-Z0-9_-]+' | head -1) || true | |
| if [ -n "$ASCIINEMA_URL" ]; then | |
| break | |
| fi | |
| if [ "$attempt" -lt "$MAX_UPLOAD_RETRIES" ]; then | |
| DELAY=$((attempt * RETRY_BASE_DELAY_SECONDS)) | |
| echo "Upload attempt $attempt failed, retrying in ${DELAY}s..." | |
| sleep "$DELAY" | |
| fi | |
| done | |
| if [ -n "$ASCIINEMA_URL" ]; then | |
| TABLE_BODY="${TABLE_BODY} | |
| | ${STATUS_EMOJI} | ${safe_filename} | [${LINK_LABEL}](${ASCIINEMA_URL}) |" | |
| echo "Uploaded: $ASCIINEMA_URL" | |
| UPLOAD_COUNT=$((UPLOAD_COUNT + 1)) | |
| # Track failed tests for the prominent section | |
| if [ "$TEST_OUTCOME" = "Failed" ]; then | |
| FAILED_TESTS_BODY="${FAILED_TESTS_BODY} | |
| - ❌ **${safe_filename}** — [${LINK_LABEL}](${ASCIINEMA_URL})" | |
| fi | |
| else | |
| TABLE_BODY="${TABLE_BODY} | |
| | ${STATUS_EMOJI} | ${safe_filename} | ⚠️ Upload failed |" | |
| echo "Failed to upload $castfile after $MAX_UPLOAD_RETRIES attempts" | |
| FAIL_COUNT=$((FAIL_COUNT + 1)) | |
| if [ "$TEST_OUTCOME" = "Failed" ]; then | |
| FAILED_TESTS_BODY="${FAILED_TESTS_BODY} | |
| - ❌ **${safe_filename}** — ⚠️ Recording upload failed" | |
| fi | |
| fi | |
| fi | |
| done | |
| echo "Uploaded $UPLOAD_COUNT recordings, $FAIL_COUNT upload failures, $TEST_PASS_COUNT passed, $TEST_FAIL_COUNT failed, $TEST_UNKNOWN_COUNT unknown" | |
| # Build the summary line in the same style as the deployment E2E comment: | |
| # "<emoji> **CLI E2E Tests <status>** — X passed, Y failed[, Z unknown]" | |
| # Status reflects test outcomes; recording-upload failures are a secondary concern | |
| # surfaced in the table rather than the headline status. | |
| # Choose headline emoji + status word from the tallied outcomes. | |
| # We never let unknowns suppress a real failure, but we do flag | |
| # unknowns explicitly when the rest of the run was clean so | |
| # reviewers don't read a misleading 'passed'. | |
| FALLBACK_TEXT="" | |
| if [ "$TEST_FAIL_COUNT" -gt 0 ]; then | |
| SUMMARY_EMOJI="❌" | |
| SUMMARY_STATUS="failed" | |
| elif [ "$TEST_PASS_COUNT" -gt 0 ] && [ "$TEST_UNKNOWN_COUNT" -eq 0 ]; then | |
| SUMMARY_EMOJI="✅" | |
| SUMMARY_STATUS="passed" | |
| elif [ "$TEST_PASS_COUNT" -eq 0 ] && [ "$TEST_FAIL_COUNT" -eq 0 ]; then | |
| # No TRX outcomes matched any recording — describe the run by | |
| # recording count rather than zero pass/fail counts, which would | |
| # read as 'no tests ran' instead of 'outcome data unavailable'. | |
| SUMMARY_EMOJI="🎬" | |
| SUMMARY_STATUS="completed" | |
| FALLBACK_TEXT="${TOTAL_COUNT} recording(s), outcomes unavailable" | |
| else | |
| SUMMARY_EMOJI="❓" | |
| SUMMARY_STATUS="unknown" | |
| fi | |
| if [ -n "$FALLBACK_TEXT" ]; then | |
| SUMMARY_TEXT="$FALLBACK_TEXT" | |
| else | |
| SUMMARY_TEXT="${TEST_PASS_COUNT} passed, ${TEST_FAIL_COUNT} failed" | |
| if [ "$TEST_UNKNOWN_COUNT" -gt 0 ]; then | |
| SUMMARY_TEXT="${SUMMARY_TEXT}, ${TEST_UNKNOWN_COUNT} unknown" | |
| fi | |
| fi | |
| if [ "$FAIL_COUNT" -gt 0 ]; then | |
| SUMMARY_TEXT="${SUMMARY_TEXT} (${UPLOAD_COUNT}/${TOTAL_COUNT} recordings uploaded, ${FAIL_COUNT} upload(s) failed)" | |
| fi | |
| # Build the failed tests section (shown outside the collapsible) | |
| FAILED_SECTION="" | |
| if [ -n "$FAILED_TESTS_BODY" ]; then | |
| FAILED_SECTION=" | |
| ### Failed Tests | |
| ${FAILED_TESTS_BODY} | |
| " | |
| fi | |
| COMMENT_BODY="${COMMENT_MARKER} | |
| ${SUMMARY_EMOJI} **CLI E2E Tests ${SUMMARY_STATUS}** — ${SUMMARY_TEXT} (commit \`${SHORT_SHA}\`) | |
| ${FAILED_SECTION} | |
| <details> | |
| <summary>View all recordings</summary> | |
| | Status | Test | Recording | | |
| |--------|------|-----------|${TABLE_BODY} | |
| --- | |
| <sub>📹 Recordings uploaded automatically from [CI run #${RUN_ID}](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID})</sub> | |
| </details>" | |
| # Delete any existing recording comments, then post the new one | |
| EXISTING_COMMENT_IDS=$(gh api graphql -f query=' | |
| query($owner: String!, $repo: String!, $pr: Int!) { | |
| repository(owner: $owner, name: $repo) { | |
| pullRequest(number: $pr) { | |
| comments(first: 100) { | |
| nodes { | |
| databaseId | |
| author { login } | |
| body | |
| } | |
| } | |
| } | |
| } | |
| }' -f owner="$GITHUB_REPOSITORY_OWNER" -f repo="$GITHUB_EVENT_REPO_NAME" -F pr="$PR_NUMBER" \ | |
| --jq '.data.repository.pullRequest.comments.nodes[] | select(.author.login == "github-actions" and (.body | contains("'"${COMMENT_MARKER}"'"))) | .databaseId') || true | |
| for COMMENT_ID in $EXISTING_COMMENT_IDS; do | |
| echo "Deleting old comment $COMMENT_ID" | |
| gh api \ | |
| --method DELETE \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "/repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" || true | |
| done | |
| echo "Creating new comment on PR #${PR_NUMBER}" | |
| gh pr comment "${PR_NUMBER}" --repo "$GITHUB_REPOSITORY" --body "$COMMENT_BODY" | |
| echo "Posted comment to PR #${PR_NUMBER}" | |
| else | |
| echo "No recordings found in $RECORDINGS_DIR" | |
| fi |