fix(fx-core): align the MCP url validation and warning delivery onto v4 #787
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR VscUse Test Plan Selector | |
| # Triggered automatically on every PR open/sync/reopen/ready_for_review, | |
| # OR manually via workflow_dispatch with a pr_number input. | |
| # | |
| # IMPORTANT: this workflow is INFORMATIONAL ONLY — it must NEVER block merge. | |
| # * The job always exits with success, regardless of UI test outcome. | |
| # * Do NOT add this workflow to required status checks in branch protection. | |
| # * Authors can merge while it is still running, or after it reports failure. | |
| # * Results are surfaced via PR comments; use them as guidance, not a gate. | |
| # | |
| # This workflow: | |
| # 1. Resolves PR context (works for both automatic and manual triggers) | |
| # 2. Gets the diff between the PR branch and the target branch | |
| # 3. Selects test plans — a changed test plan file is run as an "impacted" case; | |
| # for product-code changes GitHub Copilot CLI (Claude Sonnet 4.6) picks relevant plans | |
| # 4. Posts a PR comment listing the selected plans | |
| # 5. Dispatches dev-smoke-test-pipeline.yml with those plans | |
| # 6. Resolves (edits) the PR comment with the final pass/fail result | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'PR number to analyze and run tests for' | |
| required: true | |
| type: number | |
| # Cancel any in-progress run for the same PR if re-triggered | |
| concurrency: | |
| group: pr-vscuse-test-${{ github.event.inputs.pr_number || github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| permissions: | |
| pull-requests: write | |
| issues: write | |
| actions: write | |
| contents: read | |
| jobs: | |
| select-plans-and-run: | |
| # Run for: automatic PR events (skip drafts) or manual dispatch. | |
| if: >- | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'pull_request' && github.event.pull_request.draft == false) | |
| runs-on: ubuntu-latest | |
| steps: | |
| # ── 0. Resolve PR context (supports both pull_request and workflow_dispatch) ── | |
| - name: Resolve PR context | |
| id: pr-context | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| let prNumber, headRef, baseRef, prTitle, prBody; | |
| if (context.eventName === 'pull_request') { | |
| prNumber = context.payload.pull_request.number; | |
| headRef = context.payload.pull_request.head.ref; | |
| baseRef = context.payload.pull_request.base.ref; | |
| prTitle = context.payload.pull_request.title; | |
| prBody = context.payload.pull_request.body || ''; | |
| } else { | |
| // workflow_dispatch: fetch PR metadata from API | |
| prNumber = parseInt('${{ github.event.inputs.pr_number }}'); | |
| const { data: pr } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: prNumber, | |
| }); | |
| headRef = pr.head.ref; | |
| baseRef = pr.base.ref; | |
| prTitle = pr.title; | |
| prBody = pr.body || ''; | |
| } | |
| core.setOutput('pr_number', String(prNumber)); | |
| core.setOutput('head_ref', headRef); | |
| core.setOutput('base_ref', baseRef); | |
| core.setOutput('pr_title', prTitle); | |
| core.setOutput('pr_body', prBody.slice(0, 2000)); | |
| console.log(`PR #${prNumber}: ${headRef} -> ${baseRef} "${prTitle}"`); | |
| # ── 0b. Fetch most recent user-supplied AI hint (posted by pr-vscuse-test-comment-trigger.yml) ── | |
| - name: Fetch latest user hint from PR comments | |
| id: user-hint | |
| uses: actions/github-script@v6 | |
| env: | |
| PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }} | |
| with: | |
| script: | | |
| const MARKER = '<!-- atk-vsuse-test-hint -->'; | |
| const prNumber = parseInt(process.env.PR_NUMBER); | |
| const comments = await github.paginate(github.rest.issues.listComments, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| per_page: 100, | |
| }); | |
| // Most recent hint comment wins. | |
| const hintComment = comments | |
| .filter(c => (c.body || '').includes(MARKER)) | |
| .sort((a, b) => new Date(b.created_at) - new Date(a.created_at))[0]; | |
| let hint = ''; | |
| if (hintComment) { | |
| // Extract text inside the first fenced code block of the marker comment. | |
| const m = hintComment.body.match(/```[^\n]*\n([\s\S]*?)\n```/); | |
| hint = (m ? m[1] : hintComment.body.replace(MARKER, '')).trim(); | |
| } | |
| // Cap to keep prompt size bounded. | |
| if (hint.length > 1500) hint = hint.slice(0, 1500) + '…'; | |
| core.setOutput('hint', hint); | |
| console.log(hint | |
| ? `Found user hint (${hint.length} chars): ${hint.slice(0, 200)}` | |
| : 'No user hint comment found.'); | |
| # ── 1. Checkout plans dir + full git history for diff ───────────── | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.pr-context.outputs.head_ref }} | |
| fetch-depth: 0 | |
| sparse-checkout: | | |
| packages/tests/vscuse/vscode-test-cases/plans | |
| packages/tests/vscuse/vscode-test-cases/groups | |
| packages/tests/vscuse/smoking-test-cases.json | |
| # ── 2. Get changed files via API (PR file list) ──────────────────── | |
| - name: Get PR changed files | |
| id: changed-files | |
| uses: actions/github-script@v6 | |
| env: | |
| PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }} | |
| with: | |
| script: | | |
| const { data: files } = await github.rest.pulls.listFiles({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: parseInt(process.env.PR_NUMBER), | |
| per_page: 100, | |
| }); | |
| const fileList = files.map(f => `${f.status.padEnd(9)} ${f.filename}`).join('\n'); | |
| core.setOutput('files', fileList); | |
| console.log(`Changed files (${files.length}):\n${fileList}`); | |
| # ── 2b. Check if PR is non-code-only (test/doc/ci) — skip dispatch ── | |
| - name: Check if non-code-only PR | |
| id: code-check | |
| uses: actions/github-script@v6 | |
| env: | |
| PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }} | |
| USER_HINT: ${{ steps.user-hint.outputs.hint }} | |
| with: | |
| script: | | |
| const files = await github.paginate(github.rest.pulls.listFiles, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: parseInt(process.env.PR_NUMBER), | |
| per_page: 100, | |
| }); | |
| // Patterns that are considered non-code (test / doc / ci / config-only) | |
| const NON_CODE_PREFIXES = [ | |
| 'packages/tests/', // all test files | |
| 'docs/', // documentation | |
| 'Localize/', // localization | |
| '.github/', // CI workflows & instructions | |
| ]; | |
| const NON_CODE_SUFFIXES = [ | |
| '.md', // markdown (readme, changelog, etc.) | |
| '.lcl', // localization files | |
| ]; | |
| const NON_CODE_EXACT = [ | |
| '.gitignore', | |
| '.gitattributes', | |
| '.dockerignore', | |
| ]; | |
| const isNonCode = f => | |
| NON_CODE_PREFIXES.some(p => f.startsWith(p)) || | |
| NON_CODE_SUFFIXES.some(s => f.endsWith(s)) || | |
| NON_CODE_EXACT.includes(f); | |
| // Impacted test plans: an added / modified / renamed plan JSON maps 1:1 to a | |
| // runnable plan (basename == plan name). Removed plans cannot be run. | |
| const PLAN_DIR = 'packages/tests/vscuse/vscode-test-cases/plans/'; | |
| const GROUP_DIR = 'packages/tests/vscuse/vscode-test-cases/groups/'; | |
| const changedPlans = files | |
| .filter(f => f.filename.startsWith(PLAN_DIR) && | |
| f.filename.endsWith('.json') && | |
| f.status !== 'removed') | |
| .map(f => f.filename.slice(PLAN_DIR.length).replace(/\.json$/, '')); | |
| const changedGroups = files | |
| .filter(f => f.filename.startsWith(GROUP_DIR) && | |
| f.filename.endsWith('.json') && | |
| f.status !== 'removed') | |
| .map(f => f.filename.slice(GROUP_DIR.length).replace(/\.json$/, '')); | |
| const hasUserHint = Boolean((process.env.USER_HINT || '').trim()); | |
| const allFiles = files.map(f => f.filename); | |
| const codeFiles = allFiles.filter(f => !isNonCode(f)); | |
| // Run when product code changed OR test plans/groups changed (impacted plans) | |
| // OR a user explicitly provided an atk-vscuse-test hint. | |
| const skipDispatch = codeFiles.length === 0 && changedPlans.length === 0 && changedGroups.length === 0 && !hasUserHint; | |
| core.setOutput('skip_dispatch', String(skipDispatch)); | |
| core.setOutput('has_code_files', String(codeFiles.length > 0)); | |
| core.setOutput('has_user_hint', String(hasUserHint)); | |
| core.setOutput('changed_plans_json', JSON.stringify(changedPlans)); | |
| core.setOutput('changed_groups_json', JSON.stringify(changedGroups)); | |
| if (skipDispatch) { | |
| console.log(`All ${allFiles.length} changed files are test/doc/ci — skipping vscuse dispatch.`); | |
| } else { | |
| if (codeFiles.length) { | |
| console.log(`${codeFiles.length} code file(s) changed — vscuse dispatch required.`); | |
| codeFiles.forEach(f => console.log(` code: ${f}`)); | |
| } | |
| if (changedPlans.length) { | |
| console.log(`${changedPlans.length} impacted test plan(s) changed:`); | |
| changedPlans.forEach(p => console.log(` plan: ${p}`)); | |
| } | |
| if (changedGroups.length) { | |
| console.log(`${changedGroups.length} changed test group file(s):`); | |
| changedGroups.forEach(g => console.log(` group: ${g}`)); | |
| } | |
| if (hasUserHint) { | |
| console.log('User hint is present — forcing dispatch even if this is a non-code-only PR.'); | |
| } | |
| } | |
| # ── 2c. Post skip comment when non-code-only ─────────────────────── | |
| - name: Post skip comment (non-code-only PR) | |
| if: steps.code-check.outputs.skip_dispatch == 'true' | |
| uses: actions/github-script@v6 | |
| env: | |
| PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }} | |
| HEAD_REF: ${{ steps.pr-context.outputs.head_ref }} | |
| BASE_REF: ${{ steps.pr-context.outputs.base_ref }} | |
| with: | |
| script: | | |
| const prNumber = parseInt(process.env.PR_NUMBER); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| body: [ | |
| '## ⏭️ VscUse Test Plan — Skipped', | |
| '', | |
| `**Branch diff:** \`${process.env.HEAD_REF}\` → \`${process.env.BASE_REF}\``, | |
| '', | |
| 'All changed files are **test / doc / CI only** — no product code was modified.', | |
| 'VscUse UI tests are not required for this PR.', | |
| ].join('\n'), | |
| }); | |
| console.log(`Posted skip comment on PR #${prNumber}`); | |
| # ── 3. Get git diff summary between PR branch and target branch ──── | |
| - name: Get git diff summary | |
| id: diff-summary | |
| if: steps.code-check.outputs.skip_dispatch != 'true' && steps.code-check.outputs.has_code_files == 'true' | |
| env: | |
| BASE_REF: ${{ steps.pr-context.outputs.base_ref }} | |
| HEAD_REF: ${{ steps.pr-context.outputs.head_ref }} | |
| run: | | |
| git fetch origin "$BASE_REF" --depth=50 2>/dev/null || true | |
| git fetch origin "$HEAD_REF" --depth=50 2>/dev/null || true | |
| DIFF_STAT=$(git diff "origin/${BASE_REF}...origin/${HEAD_REF}" --stat --no-color 2>/dev/null \ | |
| | tail -5 || echo "(diff unavailable)") | |
| echo "diff_stat<<EOF" >> "$GITHUB_OUTPUT" | |
| echo "$DIFF_STAT" >> "$GITHUB_OUTPUT" | |
| echo "EOF" >> "$GITHUB_OUTPUT" | |
| # ── 3b. Install Copilot CLI ─────────────────────────────────────── | |
| - uses: actions/setup-node@v4 | |
| if: steps.code-check.outputs.skip_dispatch != 'true' && steps.code-check.outputs.has_code_files == 'true' | |
| with: | |
| node-version: '22' | |
| - name: Install Copilot CLI | |
| if: steps.code-check.outputs.skip_dispatch != 'true' && steps.code-check.outputs.has_code_files == 'true' | |
| run: | | |
| npm install -g @github/copilot@1.0.44 | |
| copilot --version | |
| # ── 4. Select test plans via Copilot CLI ───────────────────────── | |
| - name: Select test plans via AI | |
| id: ai-select | |
| if: steps.code-check.outputs.skip_dispatch != 'true' | |
| env: | |
| COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_RUNNER_TOKEN }} | |
| PR_TITLE: ${{ steps.pr-context.outputs.pr_title }} | |
| PR_BODY: ${{ steps.pr-context.outputs.pr_body }} | |
| CHANGED_FILES: ${{ steps.changed-files.outputs.files }} | |
| DIFF_STAT: ${{ steps.diff-summary.outputs.diff_stat }} | |
| BASE_BRANCH: ${{ steps.pr-context.outputs.base_ref }} | |
| HEAD_BRANCH: ${{ steps.pr-context.outputs.head_ref }} | |
| USER_HINT: ${{ steps.user-hint.outputs.hint }} | |
| CHANGED_PLANS_JSON: ${{ steps.code-check.outputs.changed_plans_json }} | |
| CHANGED_GROUPS_JSON: ${{ steps.code-check.outputs.changed_groups_json }} | |
| HAS_CODE_FILES: ${{ steps.code-check.outputs.has_code_files }} | |
| run: | | |
| set -euo pipefail | |
| # List every available plan name (one per line) | |
| PLANS_TEXT=$(find packages/tests/vscuse/vscode-test-cases/plans \ | |
| -maxdepth 1 -name "*.json" -exec basename {} .json \; | sort) | |
| # ── Impacted plans: changed test plan files map 1:1 to runnable plans. ── | |
| IMPACTED_PLANS='[]' | |
| if [ -n "${CHANGED_PLANS_JSON:-}" ] && [ "${CHANGED_PLANS_JSON:-}" != "[]" ]; then | |
| while IFS= read -r plan; do | |
| [ -z "$plan" ] && continue | |
| if [ -f "packages/tests/vscuse/vscode-test-cases/plans/${plan}.json" ]; then | |
| IMPACTED_PLANS=$(echo "$IMPACTED_PLANS" | jq --arg p "$plan" '. + [$p]') | |
| else | |
| echo "Warning: changed plan not found on disk, skipping: $plan" | |
| fi | |
| done < <(echo "$CHANGED_PLANS_JSON" | jq -r '.[]' 2>/dev/null || true) | |
| fi | |
| IMPACTED_COUNT=$(echo "$IMPACTED_PLANS" | jq 'length') | |
| echo "Impacted plans (${IMPACTED_COUNT}): $IMPACTED_PLANS" | |
| # ── Map changed groups -> impacted plans via plans' execution_order ── | |
| if [ -n "${CHANGED_GROUPS_JSON:-}" ] && [ "${CHANGED_GROUPS_JSON:-}" != "[]" ]; then | |
| while IFS= read -r group; do | |
| [ -z "$group" ] && continue | |
| group_file="packages/tests/vscuse/vscode-test-cases/groups/${group}.json" | |
| if [ ! -f "$group_file" ]; then | |
| echo "Warning: changed group not found on disk, skipping: $group" | |
| continue | |
| fi | |
| group_plan_id=$(jq -r '.plan_metadata.plan_id // empty' "$group_file" 2>/dev/null || true) | |
| if [ -z "$group_plan_id" ]; then | |
| echo "Warning: group file has no plan_metadata.plan_id, skipping: $group_file" | |
| continue | |
| fi | |
| while IFS= read -r plan_path; do | |
| [ -z "$plan_path" ] && continue | |
| plan_name=$(basename "$plan_path" .json) | |
| IMPACTED_PLANS=$(echo "$IMPACTED_PLANS" | jq --arg p "$plan_name" '. + [$p]') | |
| done < <(jq -r --arg gid "$group_plan_id" ' | |
| select((.plan_metadata.execution_order // []) | index($gid)) | input_filename | |
| ' packages/tests/vscuse/vscode-test-cases/plans/*.json 2>/dev/null || true) | |
| done < <(echo "$CHANGED_GROUPS_JSON" | jq -r '.[]' 2>/dev/null || true) | |
| fi | |
| IMPACTED_PLANS=$(echo "$IMPACTED_PLANS" | jq 'unique') | |
| IMPACTED_COUNT=$(echo "$IMPACTED_PLANS" | jq 'length') | |
| echo "Impacted plans after group mapping (${IMPACTED_COUNT}): $IMPACTED_PLANS" | |
| # ── Explicit user-hinted plans (authoritative) ── | |
| HINT_MATCHED_PLANS='[]' | |
| if [ -n "${USER_HINT:-}" ]; then | |
| USER_HINT_LC=$(echo "$USER_HINT" | tr '[:upper:]' '[:lower:]') | |
| while IFS= read -r plan; do | |
| [ -z "$plan" ] && continue | |
| PLAN_LC=$(echo "$plan" | tr '[:upper:]' '[:lower:]') | |
| if [[ "$USER_HINT_LC" == *"$PLAN_LC"* ]]; then | |
| HINT_MATCHED_PLANS=$(echo "$HINT_MATCHED_PLANS" | jq --arg p "$plan" '. + [$p]') | |
| fi | |
| done <<< "$PLANS_TEXT" | |
| HINT_MATCHED_PLANS=$(echo "$HINT_MATCHED_PLANS" | jq 'unique') | |
| fi | |
| HINT_MATCHED_COUNT=$(echo "$HINT_MATCHED_PLANS" | jq 'length') | |
| echo "User-hinted plans matched (${HINT_MATCHED_COUNT}): $HINT_MATCHED_PLANS" | |
| # ── Plan-only PR (no product code changed): run exactly the impacted plans. ── | |
| # The Copilot CLI is intentionally not installed for this path (the install | |
| # steps are gated on has_code_files), so we must not invoke it here. | |
| if [ "${HAS_CODE_FILES:-true}" != "true" ]; then | |
| if [ "$HINT_MATCHED_COUNT" -gt 0 ]; then | |
| PLANS_CSV=$(echo "$HINT_MATCHED_PLANS" | jq -r 'join(",")') | |
| echo "test_plans=${PLANS_CSV}" >> "$GITHUB_OUTPUT" | |
| echo "reason=Selected ${HINT_MATCHED_COUNT} user-requested plan(s) from authoritative hint (no product code modified)." >> "$GITHUB_OUTPUT" | |
| echo "ai_source=user-hint" >> "$GITHUB_OUTPUT" | |
| echo "is_fallback=false" >> "$GITHUB_OUTPUT" | |
| echo "Selected user-hinted plans: $PLANS_CSV" | |
| elif [ "$IMPACTED_COUNT" -gt 0 ]; then | |
| PLANS_CSV=$(echo "$IMPACTED_PLANS" | jq -r 'join(",")') | |
| echo "test_plans=${PLANS_CSV}" >> "$GITHUB_OUTPUT" | |
| echo "reason=Selected ${IMPACTED_COUNT} impacted test plan(s) changed in this PR (no product code modified)." >> "$GITHUB_OUTPUT" | |
| echo "ai_source=impacted-plans" >> "$GITHUB_OUTPUT" | |
| echo "is_fallback=false" >> "$GITHUB_OUTPUT" | |
| echo "Selected impacted plans only: $PLANS_CSV" | |
| else | |
| SMOKE_PLANS=$(jq -c '[.smokeTestCases[]]' packages/tests/vscuse/smoking-test-cases.json) | |
| PLANS_CSV=$(echo "$SMOKE_PLANS" | jq -r 'join(",")') | |
| echo "test_plans=${PLANS_CSV}" >> "$GITHUB_OUTPUT" | |
| echo "reason=No runnable changed plans found; using smoke test cases as fallback." >> "$GITHUB_OUTPUT" | |
| echo "ai_source=smoke-fallback" >> "$GITHUB_OUTPUT" | |
| echo "is_fallback=true" >> "$GITHUB_OUTPUT" | |
| echo "Fallback to smoke plans: $PLANS_CSV" | |
| fi | |
| exit 0 | |
| fi | |
| SYSTEM_PROMPT='You are a test plan selector for the Microsoft 365 Agents Toolkit (ATK) VS Code extension. Select the most relevant VscUse UI test plans for a given PR based on the changed files and PR description. IMPORTANT: If a USER HINT is provided below, treat it as AUTHORITATIVE guidance from a human reviewer — it overrides your own heuristics. The hint may name specific plans to include or exclude, or describe which feature area to focus on; honour it as long as the requested plans actually exist in the available list. IMPORTANT CONTEXT: VscUse tests only test the VS Code extension. The plan files are all for VS Code templates. There are NO C# plan files. FILE PATH MAPPING (apply in order, stop at first match): 1) templates/vs/** (Visual Studio C# templates, NOT VS Code) -> These have no VscUse test plans; return only 1-2 smoke plans (Basic_Custom_Engine_Azure_OpenAI_ts_Copilot_Remote_Debug or General_Teams_Agent_OpenAI_py_Remote_Debug) as a basic sanity check. 2) templates/vscode/ts/custom-copilot-** -> Basic_Custom_Engine_*_ts_* Remote_Debug plans only. 3) templates/vscode/js/** -> *_js_* Remote_Debug plans only. 4) templates/vscode/py/** -> *_py_* Remote_Debug plans only. 5) templates/vscode/**declarative** or **da-** -> DA_*. 6) templates/vscode/**message-extension** -> Message_Extension_*. 7) templates/vscode/**tab** -> Basic_Tab_*. 8) templates/vscode/**bot** -> Simple_Bot_*. 9) packages/vscode-extension/src/treeview/** -> Feature_TreeView_*. 10) packages/vscode-extension/** -> Feature_*. 11) trivial/infra changes (remove pkg, typo, readme, CI yml) -> 1-2 smoke plans only. LANGUAGE RULES (for vscode templates only): _ts_=TypeScript plans; _js_=JavaScript plans; _py_=Python plans. ALL plans have a language suffix (_ts_, _js_, or _py_) - never invent a plan name without one. PLAN PREFIXES: DA_*=Declarative Agent; Basic_Custom_Engine_*=Custom Engine AI bot; General_Teams_Agent_*=Teams agent; Teams_Agent_With_Data_*=AI Search/RAG; Weather_Agent_*=Weather bot; Simple_Bot_*=Simple bot; Message_Extension_*=Message extension; Basic_Tab_*=Tab app; Feature_*=VS Code UI features. Keep total 1-5 plans (or whatever the user hint asks for). Output ONLY a JSON object: {"plans": ["PlanName1"], "reason": "one sentence (mention if a user hint was applied)"}' | |
| if [ -n "${USER_HINT:-}" ]; then | |
| HINT_BLOCK=$(printf 'USER HINT (authoritative — overrides heuristics):\n%s\n\n' "$USER_HINT") | |
| else | |
| HINT_BLOCK="" | |
| fi | |
| USER_PROMPT=$(printf '%s\n\n%sPR: %s -> %s\nTitle: %s\n\nDescription (first 1200 chars):\n%s\n\nChanged files (with status):\n%s\n\nDiff summary:\n%s\n\nAvailable test plans:\n%s\n\nOutput JSON only.' \ | |
| "$SYSTEM_PROMPT" \ | |
| "$HINT_BLOCK" \ | |
| "${HEAD_BRANCH}" "${BASE_BRANCH}" "${PR_TITLE}" "${PR_BODY:0:1200}" \ | |
| "${CHANGED_FILES}" "${DIFF_STAT}" "${PLANS_TEXT}") | |
| AI_SOURCE="smoke-fallback" | |
| echo "Running Copilot CLI for test plan selection..." | |
| copilot --yolo --model claude-sonnet-4.6 --reasoning-effort high -p "$USER_PROMPT" > /tmp/copilot_output.txt 2>&1 || true | |
| echo "--- Copilot output ---" | |
| cat /tmp/copilot_output.txt || true | |
| # Extract JSON: find the first {...} block that has a "plans" key | |
| cat > /tmp/extract_json.py << 'PYEOF' | |
| import sys, re, json | |
| text = open('/tmp/copilot_output.txt').read() | |
| for pat in [r'\{[^{}]*"plans"[^{}]*\}', r'\{.*?\}']: | |
| for m in re.findall(pat, text, re.DOTALL): | |
| try: | |
| obj = json.loads(m) | |
| if 'plans' in obj: | |
| print(json.dumps(obj)) | |
| sys.exit(0) | |
| except Exception: | |
| pass | |
| print('') | |
| PYEOF | |
| AI_CONTENT=$(python3 /tmp/extract_json.py 2>/dev/null || true) | |
| echo "Extracted AI content: $AI_CONTENT" | |
| if [ -n "$AI_CONTENT" ]; then | |
| AI_SOURCE="copilot-cli" | |
| fi | |
| AI_PLANS=$(echo "$AI_CONTENT" | jq -r '.plans // empty' 2>/dev/null || true) | |
| AI_REASON=$(echo "$AI_CONTENT" | jq -r '.reason // empty' 2>/dev/null || true) | |
| # Validate: only keep plans that exist on disk | |
| VALID_PLANS='[]' | |
| if [ -n "$AI_PLANS" ] && [ "$AI_PLANS" != "null" ]; then | |
| while IFS= read -r plan; do | |
| [ -z "$plan" ] && continue | |
| if [ -f "packages/tests/vscuse/vscode-test-cases/plans/${plan}.json" ]; then | |
| VALID_PLANS=$(echo "$VALID_PLANS" | jq --arg p "$plan" '. + [$p]') | |
| else | |
| echo "Warning: AI returned unknown plan, skipping: $plan" | |
| fi | |
| done < <(echo "$AI_PLANS" | jq -r '.[]' 2>/dev/null || true) | |
| fi | |
| # Force-include impacted plans: a changed test plan must always run, on top | |
| # of the AI's code-based picks (order-preserving dedup). | |
| if [ "$IMPACTED_COUNT" -gt 0 ]; then | |
| VALID_PLANS=$(jq -cn --argjson a "$VALID_PLANS" --argjson b "$IMPACTED_PLANS" \ | |
| 'reduce ($a + $b)[] as $p ([]; if index($p) then . else . + [$p] end)') | |
| if [ -n "$AI_REASON" ]; then | |
| AI_REASON="${AI_REASON} (Also running ${IMPACTED_COUNT} impacted test plan(s) changed in this PR.)" | |
| else | |
| AI_REASON="Running ${IMPACTED_COUNT} impacted test plan(s) changed in this PR." | |
| fi | |
| fi | |
| # Fallback to smoke tests if neither AI nor impacted plans produced anything usable | |
| IS_FALLBACK="false" | |
| if [ "$(echo "$VALID_PLANS" | jq 'length')" -eq 0 ]; then | |
| echo "No valid AI plans — falling back to smoke test cases." | |
| VALID_PLANS=$(jq -c '[.smokeTestCases[]]' packages/tests/vscuse/smoking-test-cases.json) | |
| AI_REASON="AI selection unavailable (source=${AI_SOURCE}); using smoke test cases as fallback." | |
| IS_FALLBACK="true" | |
| fi | |
| PLANS_CSV=$(echo "$VALID_PLANS" | jq -r 'join(",")') | |
| echo "test_plans=${PLANS_CSV}" >> "$GITHUB_OUTPUT" | |
| echo "reason=${AI_REASON}" >> "$GITHUB_OUTPUT" | |
| echo "ai_source=${AI_SOURCE}" >> "$GITHUB_OUTPUT" | |
| echo "is_fallback=${IS_FALLBACK}" >> "$GITHUB_OUTPUT" | |
| echo "Selected: $PLANS_CSV" | |
| echo "Reason: $AI_REASON" | |
| # ── 5. Post initial PR comment ───────────────────────────────────── | |
| - name: Post PR comment with selected test plans | |
| id: post-comment | |
| if: steps.code-check.outputs.skip_dispatch != 'true' | |
| uses: actions/github-script@v6 | |
| env: | |
| TEST_PLANS: ${{ steps.ai-select.outputs.test_plans }} | |
| REASON: ${{ steps.ai-select.outputs.reason }} | |
| AI_SOURCE: ${{ steps.ai-select.outputs.ai_source }} | |
| IS_FALLBACK: ${{ steps.ai-select.outputs.is_fallback }} | |
| PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }} | |
| HEAD_REF: ${{ steps.pr-context.outputs.head_ref }} | |
| BASE_REF: ${{ steps.pr-context.outputs.base_ref }} | |
| with: | |
| script: | | |
| const plans = process.env.TEST_PLANS.split(',').filter(Boolean); | |
| const reason = process.env.REASON || '-'; | |
| const aiSource = process.env.AI_SOURCE || 'unknown'; | |
| const isFallback = process.env.IS_FALLBACK === 'true'; | |
| const prNumber = parseInt(process.env.PR_NUMBER); | |
| const headRef = process.env.HEAD_REF; | |
| const baseRef = process.env.BASE_REF; | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const planList = plans.map(p => `- \`${p}\``).join('\n'); | |
| const header = isFallback | |
| ? `## ⚠️ VscUse Test Plan — Smoke Test Fallback\n\n> **AI selection was unavailable** (tried: \`${aiSource}\`). Running default smoke tests instead.` | |
| : `## 🤖 VscUse Test Plan — AI Selected (via \`${aiSource}\`)`; | |
| const body = [ | |
| header, | |
| '', | |
| `**Why these tests:** ${reason}`, | |
| '', | |
| `**Branch diff:** \`${headRef}\` → \`${baseRef}\``, | |
| '', | |
| '**Plans queued:**', | |
| planList, | |
| '', | |
| '---', | |
| '| Step | Status |', | |
| '|------|--------|', | |
| '| 1️⃣ Build VSIX (CD) | ⏳ Running… |', | |
| '| 2️⃣ Build Docker image | ⌛ Waiting |', | |
| '| 3️⃣ Run UI tests | ⌛ Waiting |', | |
| '', | |
| `> 🔗 [View pipeline run](${runUrl})`, | |
| ].join('\n'); | |
| const { data: comment } = await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| body, | |
| }); | |
| core.setOutput('comment_id', String(comment.id)); | |
| console.log(`Posted comment: ${comment.html_url}`); | |
| # ── 6. Dispatch dev-smoke-test-pipeline with selected plans ──────── | |
| - name: Dispatch dev-smoke-test-pipeline.yml | |
| id: dispatch | |
| if: steps.code-check.outputs.skip_dispatch != 'true' | |
| uses: actions/github-script@v6 | |
| env: | |
| TEST_PLANS: ${{ steps.ai-select.outputs.test_plans }} | |
| HEAD_BRANCH: ${{ steps.pr-context.outputs.head_ref }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const testPlans = process.env.TEST_PLANS; | |
| const headBranch = process.env.HEAD_BRANCH; | |
| // dev-smoke-test-pipeline.yml on dev branch may not have test_plan yet (pending PR merge) | |
| // Try with test_plan first, fall back to without it if rejected | |
| let dispatched = false; | |
| try { | |
| await github.rest.actions.createWorkflowDispatch({ | |
| owner, | |
| repo, | |
| workflow_id: 'dev-smoke-test-pipeline.yml', | |
| ref: headBranch, | |
| inputs: { | |
| test_plan: testPlans, | |
| email: 'atk-vsuse-test@microsoft.com', | |
| }, | |
| }); | |
| dispatched = true; | |
| console.log('Dispatched with test_plan input'); | |
| } catch (err) { | |
| if (err.status === 422) { | |
| console.log('test_plan input not supported on target branch, retrying without it...'); | |
| await github.rest.actions.createWorkflowDispatch({ | |
| owner, | |
| repo, | |
| workflow_id: 'dev-smoke-test-pipeline.yml', | |
| ref: headBranch, | |
| inputs: { email: 'atk-vsuse-test@microsoft.com' }, | |
| }); | |
| dispatched = true; | |
| console.log('Dispatched without test_plan input'); | |
| } else { | |
| throw err; | |
| } | |
| } | |
| await new Promise(r => setTimeout(r, 15000)); | |
| const { data: { workflow_runs: runs } } = await github.rest.actions.listWorkflowRuns({ | |
| owner, | |
| repo, | |
| workflow_id: 'dev-smoke-test-pipeline.yml', | |
| branch: headBranch, | |
| per_page: 5, | |
| }); | |
| const run = runs.find(r => | |
| r.status !== 'completed' || | |
| (Date.now() - new Date(r.created_at).getTime()) < 60000 | |
| ); | |
| if (!run) throw new Error('Could not locate the dispatched dev-smoke-test-pipeline run'); | |
| core.setOutput('smoke_run_id', String(run.id)); | |
| core.setOutput('smoke_run_url', run.html_url); | |
| console.log(`Smoke pipeline run ID: ${run.id} URL: ${run.html_url}`); | |
| # Update comment: pipeline now running | |
| - name: Update PR comment — pipeline dispatched | |
| if: steps.code-check.outputs.skip_dispatch != 'true' | |
| uses: actions/github-script@v6 | |
| env: | |
| COMMENT_ID: ${{ steps.post-comment.outputs.comment_id }} | |
| TEST_PLANS: ${{ steps.ai-select.outputs.test_plans }} | |
| REASON: ${{ steps.ai-select.outputs.reason }} | |
| RUN_URL: ${{ steps.dispatch.outputs.smoke_run_url }} | |
| PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }} | |
| HEAD_REF: ${{ steps.pr-context.outputs.head_ref }} | |
| BASE_REF: ${{ steps.pr-context.outputs.base_ref }} | |
| with: | |
| script: | | |
| const commentId = parseInt(process.env.COMMENT_ID); | |
| const plans = process.env.TEST_PLANS.split(',').filter(Boolean); | |
| const reason = process.env.REASON || '-'; | |
| const runUrl = process.env.RUN_URL; | |
| const headRef = process.env.HEAD_REF; | |
| const baseRef = process.env.BASE_REF; | |
| const planList = plans.map(p => `- \`${p}\``).join('\n'); | |
| const body = [ | |
| '## 🤖 VscUse Test Plan — AI Selected', | |
| '', | |
| `**Why these tests:** ${reason}`, | |
| '', | |
| `**Branch diff:** \`${headRef}\` → \`${baseRef}\``, | |
| '', | |
| '**Plans queued:**', | |
| planList, | |
| '', | |
| '---', | |
| '| Step | Status |', | |
| '|------|--------|', | |
| '| 1️⃣ Build VSIX (CD) | ⏳ Running… |', | |
| '| 2️⃣ Build Docker image | ⌛ Waiting |', | |
| '| 3️⃣ Run UI tests | ⌛ Waiting |', | |
| '', | |
| `> 🔗 [View smoke pipeline](${runUrl})`, | |
| ].join('\n'); | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: commentId, | |
| body, | |
| }); | |
| # ── 7. Poll smoke pipeline until done ───────────────────────────── | |
| - name: Wait for smoke pipeline to complete | |
| id: wait-smoke | |
| if: steps.code-check.outputs.skip_dispatch != 'true' | |
| uses: actions/github-script@v6 | |
| env: | |
| SMOKE_RUN_ID: ${{ steps.dispatch.outputs.smoke_run_id }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const runId = parseInt(process.env.SMOKE_RUN_ID); | |
| const start = Date.now(); | |
| const EIGHT_HOURS = 8 * 60 * 60 * 1000; | |
| // Try to extract the actual UI-test run URL from the smoke pipeline's | |
| // `run-ui-test` job logs. That job dispatches ui-test-vscuse-template.yml | |
| // and prints "UI Test run ID: <id>" — surfacing it lets the PR comment | |
| // link straight to the real test report instead of the smoke wrapper. | |
| const extractUiTestUrl = async () => { | |
| try { | |
| const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { | |
| owner, repo, run_id: runId, per_page: 100, | |
| }); | |
| const uiJob = jobs.find(j => j.name === 'run-ui-test'); | |
| if (!uiJob) { | |
| console.log('run-ui-test job not found on smoke pipeline.'); | |
| return ''; | |
| } | |
| const logs = await github.request( | |
| 'GET /repos/{owner}/{repo}/actions/jobs/{job_id}/logs', | |
| { owner, repo, job_id: uiJob.id } | |
| ); | |
| const match = String(logs.data || '').match(/UI Test run ID:\s*(\d+)/); | |
| if (!match) { | |
| console.log('Could not find "UI Test run ID:" line in run-ui-test logs.'); | |
| return ''; | |
| } | |
| const uiRunId = match[1]; | |
| const uiUrl = `https://github.com/${owner}/${repo}/actions/runs/${uiRunId}`; | |
| console.log(`Extracted UI test run: ${uiUrl}`); | |
| return uiUrl; | |
| } catch (err) { | |
| console.log(`Failed to extract UI test run URL: ${err.message}`); | |
| return ''; | |
| } | |
| }; | |
| // On the UI test run, the `report` job (ui-test-vscuse-common.yml) uploads | |
| // the aggregated email-body.html to Azure Blob and prints | |
| // "Report Summary URL: <url>". The PR-trigger path emails the report to an | |
| // unattended mailbox, so this single aggregate blob link is the only | |
| // user-facing report surface. | |
| const extractReportUrl = async (uiUrl) => { | |
| const idMatch = (uiUrl || '').match(/\/runs\/(\d+)/); | |
| if (!idMatch) return ''; | |
| try { | |
| // Paginate: a UI-test run has one matrix job per selected test plan and | |
| // can exceed 100 jobs. The report job is near the end, so a single | |
| // per_page=100 page would miss it and drop the report link. | |
| const uiJobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { | |
| owner, repo, run_id: parseInt(idMatch[1]), per_page: 100, | |
| }); | |
| // The report job is invoked via a reusable workflow, so its rendered | |
| // name is typically "<caller_job> / report". Match the trailing token. | |
| const reportJob = uiJobs.find(j => | |
| /(^|\/\s*)report$/.test((j.name || '').trim()) | |
| ); | |
| if (!reportJob) { console.log('report job not found on UI test run.'); return ''; } | |
| const log = await github.request( | |
| 'GET /repos/{owner}/{repo}/actions/jobs/{job_id}/logs', | |
| { owner, repo, job_id: reportJob.id } | |
| ); | |
| const m = String(log.data || '').match(/Report Summary URL:\s*(https?:\/\/\S+)/); | |
| if (!m) { console.log('No "Report Summary URL:" marker found in report job logs.'); return ''; } | |
| console.log(`Extracted aggregated report URL: ${m[1]}`); | |
| return m[1]; | |
| } catch (err) { | |
| console.log(`Failed to extract aggregated report URL: ${err.message}`); | |
| return ''; | |
| } | |
| }; | |
| // Count per-plan (matrix) job outcomes on the UI test run. Each selected | |
| // plan renders as a "<caller_job> / Case-<plan>" job (see ui-test-vscuse-common.yml | |
| // job `main`, name "Case-${{ matrix.test_plan }}"). Retries re-run failed jobs as | |
| // new run attempts, so list with filter=all and keep the latest attempt per plan | |
| // to reflect each plan's final pass/fail status. | |
| const extractTestCounts = async (uiUrl) => { | |
| const idMatch = (uiUrl || '').match(/\/runs\/(\d+)/); | |
| if (!idMatch) return null; | |
| try { | |
| const uiJobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { | |
| owner, repo, run_id: parseInt(idMatch[1]), filter: 'all', per_page: 100, | |
| }); | |
| // Keep the newest attempt for each distinct Case-<plan> job. | |
| const latestByName = new Map(); | |
| for (const j of uiJobs) { | |
| const name = (j.name || '').trim(); | |
| if (!/(?:^|\/\s*)Case-[^/]+$/.test(name)) continue; | |
| const prev = latestByName.get(name); | |
| if (!prev || (j.run_attempt || 0) >= (prev.run_attempt || 0)) { | |
| latestByName.set(name, j); | |
| } | |
| } | |
| let passed = 0, failed = 0; | |
| for (const j of latestByName.values()) { | |
| if (j.conclusion === 'success') passed++; | |
| else if (j.conclusion === 'skipped' || j.conclusion == null) continue; | |
| else failed++; | |
| } | |
| const total = passed + failed; | |
| console.log(`Test-plan results: ${passed} passed, ${failed} failed (of ${total} plans).`); | |
| return { passed, failed, total }; | |
| } catch (err) { | |
| console.log(`Failed to compute test-plan counts: ${err.message}`); | |
| return null; | |
| } | |
| }; | |
| while (Date.now() - start < EIGHT_HOURS) { | |
| const { data: run } = await github.rest.actions.getWorkflowRun({ owner, repo, run_id: runId }); | |
| console.log(`Smoke pipeline: ${run.status} / ${run.conclusion}`); | |
| if (run.status === 'completed') { | |
| core.setOutput('conclusion', run.conclusion); | |
| core.setOutput('run_url', run.html_url); | |
| const uiUrl = await extractUiTestUrl(); | |
| core.setOutput('ui_test_url', uiUrl); | |
| core.setOutput('report_url', await extractReportUrl(uiUrl)); | |
| const counts = await extractTestCounts(uiUrl); | |
| core.setOutput('passed_count', counts ? String(counts.passed) : ''); | |
| core.setOutput('failed_count', counts ? String(counts.failed) : ''); | |
| core.setOutput('total_count', counts ? String(counts.total) : ''); | |
| return; | |
| } | |
| await new Promise(r => setTimeout(r, 60000)); | |
| } | |
| core.setOutput('conclusion', 'timed_out'); | |
| const uiUrl = await extractUiTestUrl(); | |
| core.setOutput('ui_test_url', uiUrl); | |
| core.setOutput('report_url', await extractReportUrl(uiUrl)); | |
| const counts = await extractTestCounts(uiUrl); | |
| core.setOutput('passed_count', counts ? String(counts.passed) : ''); | |
| core.setOutput('failed_count', counts ? String(counts.failed) : ''); | |
| core.setOutput('total_count', counts ? String(counts.total) : ''); | |
| console.log('Smoke pipeline timed out after 8 hours — reporting as timed_out (workflow stays green; this run is informational only).'); | |
| # ── 8. Resolve PR comment with final result ──────────────────────── | |
| - name: Resolve PR comment with test results | |
| if: always() && steps.code-check.outputs.skip_dispatch != 'true' | |
| uses: actions/github-script@v6 | |
| env: | |
| COMMENT_ID: ${{ steps.post-comment.outputs.comment_id }} | |
| TEST_PLANS: ${{ steps.ai-select.outputs.test_plans }} | |
| REASON: ${{ steps.ai-select.outputs.reason }} | |
| CONCLUSION: ${{ steps.wait-smoke.outputs.conclusion }} | |
| SMOKE_URL: ${{ steps.wait-smoke.outputs.run_url }} | |
| UI_TEST_URL: ${{ steps.wait-smoke.outputs.ui_test_url }} | |
| REPORT_URL: ${{ steps.wait-smoke.outputs.report_url }} | |
| PASSED_COUNT: ${{ steps.wait-smoke.outputs.passed_count }} | |
| FAILED_COUNT: ${{ steps.wait-smoke.outputs.failed_count }} | |
| TOTAL_COUNT: ${{ steps.wait-smoke.outputs.total_count }} | |
| PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }} | |
| HEAD_REF: ${{ steps.pr-context.outputs.head_ref }} | |
| BASE_REF: ${{ steps.pr-context.outputs.base_ref }} | |
| with: | |
| script: | | |
| const commentId = parseInt(process.env.COMMENT_ID); | |
| if (!commentId) { console.log('No comment ID — skipping update'); return; } | |
| const plans = (process.env.TEST_PLANS || '').split(',').filter(Boolean); | |
| const reason = process.env.REASON || '-'; | |
| const conclusion = process.env.CONCLUSION || 'unknown'; | |
| const smokeUrl = process.env.SMOKE_URL || ''; | |
| const uiTestUrl = process.env.UI_TEST_URL || ''; | |
| const reportUrl = process.env.REPORT_URL || ''; | |
| const headRef = process.env.HEAD_REF; | |
| const baseRef = process.env.BASE_REF; | |
| const planList = plans.map(p => `- \`${p}\``).join('\n'); | |
| const passedCount = process.env.PASSED_COUNT || ''; | |
| const failedCount = process.env.FAILED_COUNT || ''; | |
| const totalCount = process.env.TOTAL_COUNT || ''; | |
| const hasCounts = passedCount !== '' && failedCount !== '' && Number(totalCount) > 0; | |
| const resultsLine = hasCounts | |
| ? `**Results:** ✅ ${passedCount} passed · ❌ ${failedCount} failed (of ${totalCount} plans)` | |
| : ''; | |
| const succeeded = conclusion === 'success'; | |
| const icon = succeeded ? '✅' : '❌'; | |
| const statusText = succeeded ? 'All tests passed' : `Tests ${conclusion}`; | |
| const stepThreeText = hasCounts | |
| ? `${icon} ${statusText} (${passedCount}/${totalCount} passed)` | |
| : `${icon} ${statusText}`; | |
| const body = [ | |
| `## ${icon} VscUse Test Plan — ${statusText}`, | |
| '', | |
| `**Why these tests:** ${reason}`, | |
| '', | |
| `**Branch diff:** \`${headRef}\` → \`${baseRef}\``, | |
| '', | |
| resultsLine, | |
| '', | |
| '**Plans run:**', | |
| planList, | |
| '', | |
| '---', | |
| '| Step | Status |', | |
| '|------|--------|', | |
| '| 1️⃣ Build VSIX (CD) | ✅ Done |', | |
| '| 2️⃣ Build Docker image | ✅ Done |', | |
| `| 3️⃣ Run UI tests | ${stepThreeText} |`, | |
| '', | |
| uiTestUrl ? `> 🎯 [Actual UI test run](${uiTestUrl})` : '', | |
| smokeUrl ? `> 🔗 [Full pipeline results](${smokeUrl})` : '', | |
| reportUrl ? `> 📊 [Detailed test report](${reportUrl})` : '', | |
| '', | |
| '<details>', | |
| '<summary>ℹ️ How were these tests selected?</summary>', | |
| '', | |
| 'GitHub Copilot (Claude Sonnet 4.6, high reasoning) analysed the PR title, description, and the diff between', | |
| `\`${headRef}\` and \`${baseRef}\``, | |
| `to pick the most relevant test plans from [\`packages/tests/vscuse/Index.md\`](../blob/${baseRef}/packages/tests/vscuse/Index.md).`, | |
| '', | |
| '</details>', | |
| ].join('\n'); | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: commentId, | |
| body, | |
| }); | |
| console.log(`Resolved comment ${commentId}: ${conclusion}`); | |
| // Informational workflow: never fail the job, even on test failure or timeout. | |
| // The PR comment carries the result; merge gating must not depend on this run. | |
| if (!succeeded) { | |
| console.log(`VscUse tests concluded: ${conclusion}. See: ${smokeUrl}`); | |
| } |