Add CLI E2E Recording Comment #10187
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Add CLI E2E Recording Comment | |
| on: | |
| # Trigger when the CI workflow completes (success, failure, or cancelled) | |
| # We want to post recordings even if CI was cancelled since recordings may still exist | |
| workflow_run: | |
| workflows: ["CI"] | |
| types: | |
| - completed | |
| # Allow manual triggering for testing | |
| workflow_dispatch: | |
| inputs: | |
| run_id: | |
| description: 'Workflow run ID to download artifacts from' | |
| required: true | |
| type: number | |
| jobs: | |
| add-recording-comment: | |
| # Only run on the dotnet org and for pull requests | |
| # Note: This runs for all conclusions (success, failure, cancelled) since recordings may exist | |
| if: >- | |
| ${{ github.repository_owner == 'microsoft' && | |
| (github.event.workflow_run.event == 'pull_request' || github.event_name == 'workflow_dispatch') }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| actions: read | |
| steps: | |
| - name: Log workflow run info | |
| if: ${{ github.event_name == 'workflow_run' }} | |
| run: | | |
| echo "CI workflow conclusion: ${{ github.event.workflow_run.conclusion }}" | |
| echo "CI workflow run ID: ${{ github.event.workflow_run.id }}" | |
| - name: Get workflow run info | |
| id: run-info | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| let runId, prNumber, headSha; | |
| if (context.eventName === 'workflow_dispatch') { | |
| // Manual trigger - get run info from input | |
| runId = context.payload.inputs.run_id; | |
| const run = await github.rest.actions.getWorkflowRun({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: runId | |
| }); | |
| headSha = run.data.head_sha; | |
| // Find PR by head SHA | |
| const prs = await github.rest.pulls.list({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| head: `${context.repo.owner}:${run.data.head_branch}` | |
| }); | |
| prNumber = prs.data.length > 0 ? prs.data[0].number : null; | |
| } else { | |
| // Triggered by workflow_run | |
| runId = context.payload.workflow_run.id; | |
| headSha = context.payload.workflow_run.head_sha; | |
| // Get PR number from the workflow run | |
| const prs = context.payload.workflow_run.pull_requests; | |
| prNumber = prs && prs.length > 0 ? prs[0].number : null; | |
| } | |
| if (!prNumber) { | |
| console.log('No PR found for this workflow run, skipping comment'); | |
| core.setOutput('skip', 'true'); | |
| return; | |
| } | |
| core.setOutput('skip', 'false'); | |
| core.setOutput('run_id', runId); | |
| core.setOutput('pr_number', prNumber); | |
| core.setOutput('head_sha', headSha); | |
| console.log(`Run ID: ${runId}, PR: ${prNumber}, SHA: ${headSha}`); | |
| - name: Download CLI E2E test artifacts | |
| if: steps.run-info.outputs.skip != 'true' | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const runId = ${{ steps.run-info.outputs.run_id }}; | |
| // List ALL artifacts for the workflow run using pagination | |
| // (without pagination we only get the first page and miss CLI E2E artifacts) | |
| const allArtifacts = await github.paginate( | |
| github.rest.actions.listWorkflowRunArtifacts, | |
| { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: runId, | |
| per_page: 100 | |
| } | |
| ); | |
| console.log(`Total artifacts found: ${allArtifacts.length}`); | |
| // Filter for CLI E2E recording artifacts (simple pattern match) | |
| // These are uploaded by the run-tests workflow with name: cli-e2e-recordings-{TestName} | |
| const cliE2eArtifacts = allArtifacts.filter(a => | |
| a.name.startsWith('cli-e2e-recordings-') | |
| ); | |
| console.log(`Found ${cliE2eArtifacts.length} CLI E2E recording artifacts`); | |
| // Create recordings directory | |
| const recordingsDir = 'recordings'; | |
| fs.mkdirSync(recordingsDir, { recursive: true }); | |
| // Download each artifact | |
| for (const artifact of cliE2eArtifacts) { | |
| console.log(`Downloading ${artifact.name}...`); | |
| const download = await github.rest.actions.downloadArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: artifact.id, | |
| archive_format: 'zip' | |
| }); | |
| const artifactPath = path.join(recordingsDir, `${artifact.name}.zip`); | |
| fs.writeFileSync(artifactPath, Buffer.from(download.data)); | |
| console.log(`Saved to ${artifactPath}`); | |
| } | |
| core.setOutput('artifact_count', cliE2eArtifacts.length); | |
| - name: Extract recordings from artifacts | |
| if: steps.run-info.outputs.skip != 'true' | |
| shell: bash | |
| run: | | |
| mkdir -p cast_files | |
| mkdir -p trx_files | |
| for zipfile in recordings/*.zip; do | |
| if [ -f "$zipfile" ]; then | |
| echo "Extracting $zipfile..." | |
| unzip -o "$zipfile" -d "recordings/extracted_$(basename "$zipfile" .zip)" || true | |
| fi | |
| done | |
| # Find and copy all .cast files | |
| find recordings -name "*.cast" -exec cp {} cast_files/ \; 2>/dev/null || true | |
| # Find and copy all .trx files (test results) | |
| find recordings -name "*.trx" -exec cp {} trx_files/ \; 2>/dev/null || true | |
| echo "Found recordings:" | |
| ls -la cast_files/ || echo "No .cast files found" | |
| echo "Found TRX files:" | |
| ls -la trx_files/ || echo "No .trx files found" | |
| - name: Parse test results from TRX files | |
| if: steps.run-info.outputs.skip != 'true' && hashFiles('cast_files/*.cast') != '' | |
| id: test-results | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| # Parse TRX (XML) files to extract test method outcomes using yq (pre-installed on ubuntu-latest). | |
| # Produces a JSON map of testMethodName -> outcome for the bash comment step to consume. | |
| # When the same test appears in multiple TRX files (e.g. retries), "Failed" wins over other outcomes. | |
| if compgen -G trx_files/*.trx > /dev/null 2>&1; then | |
| echo "Parsing TRX files with yq..." | |
| # yq can read XML natively; extract testName+outcome from each UnitTestResult. | |
| # -s merges all files, producing a combined JSON array of results. | |
| # Prefer "Failed" over other outcomes when duplicates exist. | |
| yq -p xml -o json '.TestRun.Results.UnitTestResult | (if type == "!!seq" then .[] else . end) | {(."@testName" // ."+@testName"): (."@outcome" // ."+@outcome")}' trx_files/*.trx \ | |
| | jq -s ' | |
| reduce (.[] | to_entries[]) as {$key, $value} ({}; | |
| # Extract simple method name (last segment after dot) for .cast file matching | |
| ($key | split(".") | last) as $method | | |
| # Prefer "Failed" over any other outcome | |
| if .[$method] == "Failed" then . else .[$method] = $value end | | |
| if .[$key] == "Failed" then . else .[$key] = $value end | |
| ) | |
| ' > test_outcomes.json | |
| OUTCOME_COUNT=$(jq 'length' test_outcomes.json) | |
| echo "Parsed $OUTCOME_COUNT test outcome(s)" | |
| echo "has_outcomes=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "No TRX files found" | |
| echo '{}' > test_outcomes.json | |
| echo "has_outcomes=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Upload recordings and post comment | |
| if: steps.run-info.outputs.skip != 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} | |
| GITHUB_EVENT_REPO_NAME: ${{ github.event.repository.name }} | |
| shell: bash | |
| run: | | |
| PR_NUMBER="${{ steps.run-info.outputs.pr_number }}" | |
| RUN_ID="${{ steps.run-info.outputs.run_id }}" | |
| HEAD_SHA="${{ steps.run-info.outputs.head_sha }}" | |
| SHORT_SHA="${HEAD_SHA:0:7}" | |
| RECORDINGS_DIR="cast_files" | |
| if [ -d "$RECORDINGS_DIR" ] && compgen -G "$RECORDINGS_DIR"/*.cast > /dev/null; then | |
| # Install asciinema | |
| pip install asciinema | |
| # Load test outcomes from TRX parsing step (JSON: {"methodName": "Passed|Failed", ...}) | |
| HAS_OUTCOMES="${{ steps.test-results.outputs.has_outcomes }}" | |
| if [ "$HAS_OUTCOMES" = "true" ] && [ -f "test_outcomes.json" ]; then | |
| echo "Loaded test outcomes from TRX files" | |
| else | |
| echo "No test outcomes available, will show recordings without pass/fail status" | |
| echo '{}' > test_outcomes.json | |
| fi | |
| # Unique marker to identify CLI E2E recording comments | |
| COMMENT_MARKER="<!-- cli-e2e-recordings -->" | |
| # Retry configuration for asciinema uploads | |
| MAX_UPLOAD_RETRIES=5 | |
| RETRY_BASE_DELAY_SECONDS=30 | |
| UPLOAD_COUNT=0 | |
| FAIL_COUNT=0 | |
| TOTAL_COUNT=0 | |
| TEST_FAIL_COUNT=0 | |
| # Arrays to track failed test recordings separately | |
| FAILED_TESTS_BODY="" | |
| TABLE_BODY="" | |
| for castfile in "$RECORDINGS_DIR"/*.cast; do | |
| if [ -f "$castfile" ]; then | |
| filename=$(basename "$castfile" .cast) | |
| echo "Uploading $castfile..." | |
| TOTAL_COUNT=$((TOTAL_COUNT + 1)) | |
| # Sanitize filename for safe markdown rendering. | |
| # .cast files are named via [CallerMemberName] so should be valid C# identifiers, | |
| # but we sanitize defensively since this runs in a privileged workflow_run context | |
| # and artifacts could come from fork PRs. | |
| safe_filename=$(echo "$filename" | tr -cd 'A-Za-z0-9_.-') | |
| # Look up test outcome from TRX data. | |
| # .cast files are named after the test method name (via [CallerMemberName] in CreateTestTerminal), | |
| # so the filename matches the method name key in the outcomes JSON. | |
| TEST_OUTCOME=$(jq -r --arg name "$filename" '.[$name] // "Unknown"' test_outcomes.json) | |
| if [ "$TEST_OUTCOME" = "Passed" ]; then | |
| STATUS_EMOJI="✅" | |
| elif [ "$TEST_OUTCOME" = "Failed" ]; then | |
| STATUS_EMOJI="❌" | |
| TEST_FAIL_COUNT=$((TEST_FAIL_COUNT + 1)) | |
| else | |
| STATUS_EMOJI="❔" | |
| fi | |
| # Upload to asciinema with retry logic for transient failures | |
| ASCIINEMA_URL="" | |
| for attempt in $(seq 1 "$MAX_UPLOAD_RETRIES"); do | |
| UPLOAD_OUTPUT=$(asciinema upload "$castfile" 2>&1) || true | |
| ASCIINEMA_URL=$(echo "$UPLOAD_OUTPUT" | grep -oP 'https://asciinema\.org/a/[a-zA-Z0-9_-]+' | head -1) || true | |
| if [ -n "$ASCIINEMA_URL" ]; then | |
| break | |
| fi | |
| if [ "$attempt" -lt "$MAX_UPLOAD_RETRIES" ]; then | |
| DELAY=$((attempt * RETRY_BASE_DELAY_SECONDS)) | |
| echo "Upload attempt $attempt failed, retrying in ${DELAY}s..." | |
| sleep "$DELAY" | |
| fi | |
| done | |
| if [ -n "$ASCIINEMA_URL" ]; then | |
| TABLE_BODY="${TABLE_BODY} | |
| | ${STATUS_EMOJI} | ${safe_filename} | [▶️ View Recording](${ASCIINEMA_URL}) |" | |
| echo "Uploaded: $ASCIINEMA_URL" | |
| UPLOAD_COUNT=$((UPLOAD_COUNT + 1)) | |
| # Track failed tests for the prominent section | |
| if [ "$TEST_OUTCOME" = "Failed" ]; then | |
| FAILED_TESTS_BODY="${FAILED_TESTS_BODY} | |
| - ❌ **${safe_filename}** — [▶️ View Recording](${ASCIINEMA_URL})" | |
| fi | |
| else | |
| TABLE_BODY="${TABLE_BODY} | |
| | ${STATUS_EMOJI} | ${safe_filename} | ⚠️ Upload failed |" | |
| echo "Failed to upload $castfile after $MAX_UPLOAD_RETRIES attempts" | |
| FAIL_COUNT=$((FAIL_COUNT + 1)) | |
| if [ "$TEST_OUTCOME" = "Failed" ]; then | |
| FAILED_TESTS_BODY="${FAILED_TESTS_BODY} | |
| - ❌ **${safe_filename}** — ⚠️ Recording upload failed" | |
| fi | |
| fi | |
| fi | |
| done | |
| echo "Uploaded $UPLOAD_COUNT recordings, $FAIL_COUNT upload failures, $TEST_FAIL_COUNT test failures" | |
| # Build comment with summary outside collapsible and table inside | |
| if [ "$TEST_FAIL_COUNT" -gt 0 ]; then | |
| SUMMARY_EMOJI="❌" | |
| SUMMARY_TEXT="${TEST_FAIL_COUNT} test(s) failed, ${UPLOAD_COUNT} recordings uploaded" | |
| elif [ "$FAIL_COUNT" -gt 0 ]; then | |
| SUMMARY_EMOJI="⚠️" | |
| SUMMARY_TEXT="${UPLOAD_COUNT}/${TOTAL_COUNT} recordings uploaded, ${FAIL_COUNT} upload(s) failed" | |
| else | |
| SUMMARY_EMOJI="🎬" | |
| SUMMARY_TEXT="${UPLOAD_COUNT} recordings uploaded" | |
| fi | |
| # Build the failed tests section (shown outside the collapsible) | |
| FAILED_SECTION="" | |
| if [ -n "$FAILED_TESTS_BODY" ]; then | |
| FAILED_SECTION=" | |
| ### Failed Tests | |
| ${FAILED_TESTS_BODY} | |
| " | |
| fi | |
| COMMENT_BODY="${COMMENT_MARKER} | |
| ${SUMMARY_EMOJI} **CLI E2E Test Recordings** — ${SUMMARY_TEXT} (commit \`${SHORT_SHA}\`) | |
| ${FAILED_SECTION} | |
| <details> | |
| <summary>View all recordings</summary> | |
| | Status | Test | Recording | | |
| |--------|------|-----------|${TABLE_BODY} | |
| --- | |
| <sub>📹 Recordings uploaded automatically from [CI run #${RUN_ID}](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID})</sub> | |
| </details>" | |
| # Delete any existing recording comments, then post the new one | |
| EXISTING_COMMENT_IDS=$(gh api graphql -f query=' | |
| query($owner: String!, $repo: String!, $pr: Int!) { | |
| repository(owner: $owner, name: $repo) { | |
| pullRequest(number: $pr) { | |
| comments(first: 100) { | |
| nodes { | |
| databaseId | |
| author { login } | |
| body | |
| } | |
| } | |
| } | |
| } | |
| }' -f owner="$GITHUB_REPOSITORY_OWNER" -f repo="$GITHUB_EVENT_REPO_NAME" -F pr="$PR_NUMBER" \ | |
| --jq '.data.repository.pullRequest.comments.nodes[] | select(.author.login == "github-actions" and (.body | contains("'"${COMMENT_MARKER}"'"))) | .databaseId') || true | |
| for COMMENT_ID in $EXISTING_COMMENT_IDS; do | |
| echo "Deleting old comment $COMMENT_ID" | |
| gh api \ | |
| --method DELETE \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "/repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" || true | |
| done | |
| echo "Creating new comment on PR #${PR_NUMBER}" | |
| gh pr comment "${PR_NUMBER}" --repo "$GITHUB_REPOSITORY" --body "$COMMENT_BODY" | |
| echo "Posted comment to PR #${PR_NUMBER}" | |
| else | |
| echo "No recordings found in $RECORDINGS_DIR" | |
| fi |