refactor: update click input box description #771

Workflow file for this run

.github/workflows/pr-vscuse-test.yml at ccd1588

	name: PR VscUse Test Plan Selector

	# Triggered automatically on every PR open/sync/reopen/ready_for_review,
	# OR manually via workflow_dispatch with a pr_number input.
	#
	# IMPORTANT: this workflow is INFORMATIONAL ONLY — it must NEVER block merge.
	# * The job always exits with success, regardless of UI test outcome.
	# * Do NOT add this workflow to required status checks in branch protection.
	# * Authors can merge while it is still running, or after it reports failure.
	# * Results are surfaced via PR comments; use them as guidance, not a gate.
	#
	# This workflow:
	# 1. Resolves PR context (works for both automatic and manual triggers)
	# 2. Gets the diff between the PR branch and the target branch
	# 3. Selects test plans — a changed test plan file is run as an "impacted" case;
	# for product-code changes GitHub Copilot CLI (Claude Sonnet 4.6) picks relevant plans
	# 4. Posts a PR comment listing the selected plans
	# 5. Dispatches dev-smoke-test-pipeline.yml with those plans
	# 6. Resolves (edits) the PR comment with the final pass/fail result

	on:
	pull_request:
	types: [opened, synchronize, reopened, ready_for_review]
	workflow_dispatch:
	inputs:
	pr_number:
	description: 'PR number to analyze and run tests for'
	required: true
	type: number

	# Cancel any in-progress run for the same PR if re-triggered
	concurrency:
	group: pr-vscuse-test-${{ github.event.inputs.pr_number \|\| github.event.pull_request.number }}
	cancel-in-progress: true

	permissions:
	pull-requests: write
	issues: write
	actions: write
	contents: read

	jobs:
	select-plans-and-run:
	# Run for: automatic PR events (skip drafts) or manual dispatch.
	if: >-
	github.event_name == 'workflow_dispatch' \|\|
	(github.event_name == 'pull_request' && github.event.pull_request.draft == false)
	runs-on: ubuntu-latest

	steps:
	# ── 0. Resolve PR context (supports both pull_request and workflow_dispatch) ──
	- name: Resolve PR context
	id: pr-context
	uses: actions/github-script@v6
	with:
	script: \|
	let prNumber, headRef, baseRef, prTitle, prBody;
	if (context.eventName === 'pull_request') {
	prNumber = context.payload.pull_request.number;
	headRef = context.payload.pull_request.head.ref;
	baseRef = context.payload.pull_request.base.ref;
	prTitle = context.payload.pull_request.title;
	prBody = context.payload.pull_request.body \|\| '';
	} else {
	// workflow_dispatch: fetch PR metadata from API
	prNumber = parseInt('${{ github.event.inputs.pr_number }}');
	const { data: pr } = await github.rest.pulls.get({
	owner: context.repo.owner,
	repo: context.repo.repo,
	pull_number: prNumber,
	});
	headRef = pr.head.ref;
	baseRef = pr.base.ref;
	prTitle = pr.title;
	prBody = pr.body \|\| '';
	}
	core.setOutput('pr_number', String(prNumber));
	core.setOutput('head_ref', headRef);
	core.setOutput('base_ref', baseRef);
	core.setOutput('pr_title', prTitle);
	core.setOutput('pr_body', prBody.slice(0, 2000));
	console.log(`PR #${prNumber}: ${headRef} -> ${baseRef} "${prTitle}"`);

	# ── 0b. Fetch most recent user-supplied AI hint (posted by pr-vscuse-test-comment-trigger.yml) ──
	- name: Fetch latest user hint from PR comments
	id: user-hint
	uses: actions/github-script@v6
	env:
	PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }}
	with:
	script: \|
	const MARKER = '<!-- atk-vsuse-test-hint -->';
	const prNumber = parseInt(process.env.PR_NUMBER);
	const comments = await github.paginate(github.rest.issues.listComments, {
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: prNumber,
	per_page: 100,
	});
	// Most recent hint comment wins.
	const hintComment = comments
	.filter(c => (c.body \|\| '').includes(MARKER))
	.sort((a, b) => new Date(b.created_at) - new Date(a.created_at))[0];

	let hint = '';
	if (hintComment) {
	// Extract text inside the first fenced code block of the marker comment.
	const m = hintComment.body.match(/```[^\n]\n([\s\S]?)\n```/);
	hint = (m ? m[1] : hintComment.body.replace(MARKER, '')).trim();
	}
	// Cap to keep prompt size bounded.
	if (hint.length > 1500) hint = hint.slice(0, 1500) + '…';
	core.setOutput('hint', hint);
	console.log(hint
	? `Found user hint (${hint.length} chars): ${hint.slice(0, 200)}`
	: 'No user hint comment found.');

	# ── 1. Checkout plans dir + full git history for diff ─────────────
	- name: Checkout repo
	uses: actions/checkout@v4
	with:
	ref: ${{ steps.pr-context.outputs.head_ref }}
	fetch-depth: 0
	sparse-checkout: \|
	packages/tests/vscuse/vscode-test-cases/plans
	packages/tests/vscuse/vscode-test-cases/groups
	packages/tests/vscuse/smoking-test-cases.json

	# ── 2. Get changed files via API (PR file list) ────────────────────
	- name: Get PR changed files
	id: changed-files
	uses: actions/github-script@v6
	env:
	PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }}
	with:
	script: \|
	const { data: files } = await github.rest.pulls.listFiles({
	owner: context.repo.owner,
	repo: context.repo.repo,
	pull_number: parseInt(process.env.PR_NUMBER),
	per_page: 100,
	});
	const fileList = files.map(f => `${f.status.padEnd(9)} ${f.filename}`).join('\n');
	core.setOutput('files', fileList);
	console.log(`Changed files (${files.length}):\n${fileList}`);

	# ── 2b. Check if PR is non-code-only (test/doc/ci) — skip dispatch ──
	- name: Check if non-code-only PR
	id: code-check
	uses: actions/github-script@v6
	env:
	PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }}
	USER_HINT: ${{ steps.user-hint.outputs.hint }}
	with:
	script: \|
	const files = await github.paginate(github.rest.pulls.listFiles, {
	owner: context.repo.owner,
	repo: context.repo.repo,
	pull_number: parseInt(process.env.PR_NUMBER),
	per_page: 100,
	});

	// Patterns that are considered non-code (test / doc / ci / config-only)
	const NON_CODE_PREFIXES = [
	'packages/tests/', // all test files
	'docs/', // documentation
	'Localize/', // localization
	'.github/', // CI workflows & instructions
	];
	const NON_CODE_SUFFIXES = [
	'.md', // markdown (readme, changelog, etc.)
	'.lcl', // localization files
	];
	const NON_CODE_EXACT = [
	'.gitignore',
	'.gitattributes',
	'.dockerignore',
	];

	const isNonCode = f =>
	NON_CODE_PREFIXES.some(p => f.startsWith(p)) \|\|
	NON_CODE_SUFFIXES.some(s => f.endsWith(s)) \|\|
	NON_CODE_EXACT.includes(f);

	// Impacted test plans: an added / modified / renamed plan JSON maps 1:1 to a
	// runnable plan (basename == plan name). Removed plans cannot be run.
	const PLAN_DIR = 'packages/tests/vscuse/vscode-test-cases/plans/';
	const GROUP_DIR = 'packages/tests/vscuse/vscode-test-cases/groups/';
	const changedPlans = files
	.filter(f => f.filename.startsWith(PLAN_DIR) &&
	f.filename.endsWith('.json') &&
	f.status !== 'removed')
	.map(f => f.filename.slice(PLAN_DIR.length).replace(/\.json$/, ''));
	const changedGroups = files
	.filter(f => f.filename.startsWith(GROUP_DIR) &&
	f.filename.endsWith('.json') &&
	f.status !== 'removed')
	.map(f => f.filename.slice(GROUP_DIR.length).replace(/\.json$/, ''));

	const hasUserHint = Boolean((process.env.USER_HINT \|\| '').trim());
	const allFiles = files.map(f => f.filename);
	const codeFiles = allFiles.filter(f => !isNonCode(f));
	// Run when product code changed OR test plans/groups changed (impacted plans)
	// OR a user explicitly provided an atk-vscuse-test hint.
	const skipDispatch = codeFiles.length === 0 && changedPlans.length === 0 && changedGroups.length === 0 && !hasUserHint;

	core.setOutput('skip_dispatch', String(skipDispatch));
	core.setOutput('has_code_files', String(codeFiles.length > 0));
	core.setOutput('has_user_hint', String(hasUserHint));
	core.setOutput('changed_plans_json', JSON.stringify(changedPlans));
	core.setOutput('changed_groups_json', JSON.stringify(changedGroups));

	if (skipDispatch) {
	console.log(`All ${allFiles.length} changed files are test/doc/ci — skipping vscuse dispatch.`);
	} else {
	if (codeFiles.length) {
	console.log(`${codeFiles.length} code file(s) changed — vscuse dispatch required.`);
	codeFiles.forEach(f => console.log(` code: ${f}`));
	}
	if (changedPlans.length) {
	console.log(`${changedPlans.length} impacted test plan(s) changed:`);
	changedPlans.forEach(p => console.log(` plan: ${p}`));
	}
	if (changedGroups.length) {
	console.log(`${changedGroups.length} changed test group file(s):`);
	changedGroups.forEach(g => console.log(` group: ${g}`));
	}
	if (hasUserHint) {
	console.log('User hint is present — forcing dispatch even if this is a non-code-only PR.');
	}
	}

	# ── 2c. Post skip comment when non-code-only ───────────────────────
	- name: Post skip comment (non-code-only PR)
	if: steps.code-check.outputs.skip_dispatch == 'true'
	uses: actions/github-script@v6
	env:
	PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }}
	HEAD_REF: ${{ steps.pr-context.outputs.head_ref }}
	BASE_REF: ${{ steps.pr-context.outputs.base_ref }}
	with:
	script: \|
	const prNumber = parseInt(process.env.PR_NUMBER);
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: prNumber,
	body: [
	'## ⏭️ VscUse Test Plan — Skipped',
	'',
	`Branch diff: \`${process.env.HEAD_REF}\` → \`${process.env.BASE_REF}\``,
	'',
	'All changed files are test / doc / CI only — no product code was modified.',
	'VscUse UI tests are not required for this PR.',
	].join('\n'),
	});
	console.log(`Posted skip comment on PR #${prNumber}`);

	# ── 3. Get git diff summary between PR branch and target branch ────
	- name: Get git diff summary
	id: diff-summary
	if: steps.code-check.outputs.skip_dispatch != 'true' && steps.code-check.outputs.has_code_files == 'true'
	env:
	BASE_REF: ${{ steps.pr-context.outputs.base_ref }}
	HEAD_REF: ${{ steps.pr-context.outputs.head_ref }}
	run: \|
	git fetch origin "$BASE_REF" --depth=50 2>/dev/null \|\| true
	git fetch origin "$HEAD_REF" --depth=50 2>/dev/null \|\| true
	DIFF_STAT=$(git diff "origin/${BASE_REF}...origin/${HEAD_REF}" --stat --no-color 2>/dev/null \
	\| tail -5 \|\| echo "(diff unavailable)")
	echo "diff_stat<<EOF" >> "$GITHUB_OUTPUT"
	echo "$DIFF_STAT" >> "$GITHUB_OUTPUT"
	echo "EOF" >> "$GITHUB_OUTPUT"

	# ── 3b. Install Copilot CLI ───────────────────────────────────────
	- uses: actions/setup-node@v4
	if: steps.code-check.outputs.skip_dispatch != 'true' && steps.code-check.outputs.has_code_files == 'true'
	with:
	node-version: '22'

	- name: Install Copilot CLI
	if: steps.code-check.outputs.skip_dispatch != 'true' && steps.code-check.outputs.has_code_files == 'true'
	run: \|
	npm install -g @github/copilot@1.0.44
	copilot --version
	# ── 4. Select test plans via Copilot CLI ─────────────────────────
	- name: Select test plans via AI
	id: ai-select
	if: steps.code-check.outputs.skip_dispatch != 'true'
	env:
	COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_RUNNER_TOKEN }}
	PR_TITLE: ${{ steps.pr-context.outputs.pr_title }}
	PR_BODY: ${{ steps.pr-context.outputs.pr_body }}
	CHANGED_FILES: ${{ steps.changed-files.outputs.files }}
	DIFF_STAT: ${{ steps.diff-summary.outputs.diff_stat }}
	BASE_BRANCH: ${{ steps.pr-context.outputs.base_ref }}
	HEAD_BRANCH: ${{ steps.pr-context.outputs.head_ref }}
	USER_HINT: ${{ steps.user-hint.outputs.hint }}
	CHANGED_PLANS_JSON: ${{ steps.code-check.outputs.changed_plans_json }}
	CHANGED_GROUPS_JSON: ${{ steps.code-check.outputs.changed_groups_json }}
	HAS_CODE_FILES: ${{ steps.code-check.outputs.has_code_files }}
	run: \|
	set -euo pipefail

	# List every available plan name (one per line)
	PLANS_TEXT=$(find packages/tests/vscuse/vscode-test-cases/plans \
	-maxdepth 1 -name "*.json" -exec basename {} .json \; \| sort)

	# ── Impacted plans: changed test plan files map 1:1 to runnable plans. ──
	IMPACTED_PLANS='[]'
	if [ -n "${CHANGED_PLANS_JSON:-}" ] && [ "${CHANGED_PLANS_JSON:-}" != "[]" ]; then
	while IFS= read -r plan; do
	[ -z "$plan" ] && continue
	if [ -f "packages/tests/vscuse/vscode-test-cases/plans/${plan}.json" ]; then
	IMPACTED_PLANS=$(echo "$IMPACTED_PLANS" \| jq --arg p "$plan" '. + [$p]')
	else
	echo "Warning: changed plan not found on disk, skipping: $plan"
	fi
	done < <(echo "$CHANGED_PLANS_JSON" \| jq -r '.[]' 2>/dev/null \|\| true)
	fi
	IMPACTED_COUNT=$(echo "$IMPACTED_PLANS" \| jq 'length')
	echo "Impacted plans (${IMPACTED_COUNT}): $IMPACTED_PLANS"

	# ── Map changed groups -> impacted plans via plans' execution_order ──
	if [ -n "${CHANGED_GROUPS_JSON:-}" ] && [ "${CHANGED_GROUPS_JSON:-}" != "[]" ]; then
	while IFS= read -r group; do
	[ -z "$group" ] && continue
	group_file="packages/tests/vscuse/vscode-test-cases/groups/${group}.json"
	if [ ! -f "$group_file" ]; then
	echo "Warning: changed group not found on disk, skipping: $group"
	continue
	fi

	group_plan_id=$(jq -r '.plan_metadata.plan_id // empty' "$group_file" 2>/dev/null \|\| true)
	if [ -z "$group_plan_id" ]; then
	echo "Warning: group file has no plan_metadata.plan_id, skipping: $group_file"
	continue
	fi

	while IFS= read -r plan_path; do
	[ -z "$plan_path" ] && continue
	plan_name=$(basename "$plan_path" .json)
	IMPACTED_PLANS=$(echo "$IMPACTED_PLANS" \| jq --arg p "$plan_name" '. + [$p]')
	done < <(jq -r --arg gid "$group_plan_id" '
	select((.plan_metadata.execution_order // []) \| index($gid)) \| input_filename
	' packages/tests/vscuse/vscode-test-cases/plans/*.json 2>/dev/null \|\| true)
	done < <(echo "$CHANGED_GROUPS_JSON" \| jq -r '.[]' 2>/dev/null \|\| true)
	fi
	IMPACTED_PLANS=$(echo "$IMPACTED_PLANS" \| jq 'unique')
	IMPACTED_COUNT=$(echo "$IMPACTED_PLANS" \| jq 'length')
	echo "Impacted plans after group mapping (${IMPACTED_COUNT}): $IMPACTED_PLANS"

	# ── Explicit user-hinted plans (authoritative) ──
	HINT_MATCHED_PLANS='[]'
	if [ -n "${USER_HINT:-}" ]; then
	USER_HINT_LC=$(echo "$USER_HINT" \| tr '[:upper:]' '[:lower:]')
	while IFS= read -r plan; do
	[ -z "$plan" ] && continue
	PLAN_LC=$(echo "$plan" \| tr '[:upper:]' '[:lower:]')
	if [[ "$USER_HINT_LC" == "$PLAN_LC" ]]; then
	HINT_MATCHED_PLANS=$(echo "$HINT_MATCHED_PLANS" \| jq --arg p "$plan" '. + [$p]')
	fi
	done <<< "$PLANS_TEXT"
	HINT_MATCHED_PLANS=$(echo "$HINT_MATCHED_PLANS" \| jq 'unique')
	fi
	HINT_MATCHED_COUNT=$(echo "$HINT_MATCHED_PLANS" \| jq 'length')
	echo "User-hinted plans matched (${HINT_MATCHED_COUNT}): $HINT_MATCHED_PLANS"

	# ── Plan-only PR (no product code changed): run exactly the impacted plans. ──
	# The Copilot CLI is intentionally not installed for this path (the install
	# steps are gated on has_code_files), so we must not invoke it here.
	if [ "${HAS_CODE_FILES:-true}" != "true" ]; then
	if [ "$HINT_MATCHED_COUNT" -gt 0 ]; then
	PLANS_CSV=$(echo "$HINT_MATCHED_PLANS" \| jq -r 'join(",")')
	echo "test_plans=${PLANS_CSV}" >> "$GITHUB_OUTPUT"
	echo "reason=Selected ${HINT_MATCHED_COUNT} user-requested plan(s) from authoritative hint (no product code modified)." >> "$GITHUB_OUTPUT"
	echo "ai_source=user-hint" >> "$GITHUB_OUTPUT"
	echo "is_fallback=false" >> "$GITHUB_OUTPUT"
	echo "Selected user-hinted plans: $PLANS_CSV"
	elif [ "$IMPACTED_COUNT" -gt 0 ]; then
	PLANS_CSV=$(echo "$IMPACTED_PLANS" \| jq -r 'join(",")')
	echo "test_plans=${PLANS_CSV}" >> "$GITHUB_OUTPUT"
	echo "reason=Selected ${IMPACTED_COUNT} impacted test plan(s) changed in this PR (no product code modified)." >> "$GITHUB_OUTPUT"
	echo "ai_source=impacted-plans" >> "$GITHUB_OUTPUT"
	echo "is_fallback=false" >> "$GITHUB_OUTPUT"
	echo "Selected impacted plans only: $PLANS_CSV"
	else
	SMOKE_PLANS=$(jq -c '[.smokeTestCases[]]' packages/tests/vscuse/smoking-test-cases.json)
	PLANS_CSV=$(echo "$SMOKE_PLANS" \| jq -r 'join(",")')
	echo "test_plans=${PLANS_CSV}" >> "$GITHUB_OUTPUT"
	echo "reason=No runnable changed plans found; using smoke test cases as fallback." >> "$GITHUB_OUTPUT"
	echo "ai_source=smoke-fallback" >> "$GITHUB_OUTPUT"
	echo "is_fallback=true" >> "$GITHUB_OUTPUT"
	echo "Fallback to smoke plans: $PLANS_CSV"
	fi
	exit 0
	fi

	SYSTEM_PROMPT='You are a test plan selector for the Microsoft 365 Agents Toolkit (ATK) VS Code extension. Select the most relevant VscUse UI test plans for a given PR based on the changed files and PR description. IMPORTANT: If a USER HINT is provided below, treat it as AUTHORITATIVE guidance from a human reviewer — it overrides your own heuristics. The hint may name specific plans to include or exclude, or describe which feature area to focus on; honour it as long as the requested plans actually exist in the available list. IMPORTANT CONTEXT: VscUse tests only test the VS Code extension. The plan files are all for VS Code templates. There are NO C# plan files. FILE PATH MAPPING (apply in order, stop at first match): 1) templates/vs/ (Visual Studio C# templates, NOT VS Code) -> These have no VscUse test plans; return only 1-2 smoke plans (Basic_Custom_Engine_Azure_OpenAI_ts_Copilot_Remote_Debug or General_Teams_Agent_OpenAI_py_Remote_Debug) as a basic sanity check. 2) templates/vscode/ts/custom-copilot- -> Basic_Custom_Engine__ts_ Remote_Debug plans only. 3) templates/vscode/js/** -> _js_ Remote_Debug plans only. 4) templates/vscode/py/** -> _py_ Remote_Debug plans only. 5) templates/vscode/declarative or da- -> DA_. 6) templates/vscode/message-extension* -> Message_Extension_. 7) templates/vscode/tab* -> Basic_Tab_. 8) templates/vscode/bot* -> Simple_Bot_. 9) packages/vscode-extension/src/treeview/* -> Feature_TreeView_. 10) packages/vscode-extension/* -> Feature_. 11) trivial/infra changes (remove pkg, typo, readme, CI yml) -> 1-2 smoke plans only. LANGUAGE RULES (for vscode templates only): _ts_=TypeScript plans; _js_=JavaScript plans; _py_=Python plans. ALL plans have a language suffix (_ts_, _js_, or _py_) - never invent a plan name without one. PLAN PREFIXES: DA_=Declarative Agent; Basic_Custom_Engine_=Custom Engine AI bot; General_Teams_Agent_=Teams agent; Teams_Agent_With_Data_=AI Search/RAG; Weather_Agent_=Weather bot; Simple_Bot_=Simple bot; Message_Extension_=Message extension; Basic_Tab_=Tab app; Feature_=VS Code UI features. Keep total 1-5 plans (or whatever the user hint asks for). Output ONLY a JSON object: {"plans": ["PlanName1"], "reason": "one sentence (mention if a user hint was applied)"}'

	if [ -n "${USER_HINT:-}" ]; then
	HINT_BLOCK=$(printf 'USER HINT (authoritative — overrides heuristics):\n%s\n\n' "$USER_HINT")
	else
	HINT_BLOCK=""
	fi

	USER_PROMPT=$(printf '%s\n\n%sPR: %s -> %s\nTitle: %s\n\nDescription (first 1200 chars):\n%s\n\nChanged files (with status):\n%s\n\nDiff summary:\n%s\n\nAvailable test plans:\n%s\n\nOutput JSON only.' \
	"$SYSTEM_PROMPT" \
	"$HINT_BLOCK" \
	"${HEAD_BRANCH}" "${BASE_BRANCH}" "${PR_TITLE}" "${PR_BODY:0:1200}" \
	"${CHANGED_FILES}" "${DIFF_STAT}" "${PLANS_TEXT}")

	AI_SOURCE="smoke-fallback"
	echo "Running Copilot CLI for test plan selection..."
	copilot --yolo --model claude-sonnet-4.6 --reasoning-effort high -p "$USER_PROMPT" > /tmp/copilot_output.txt 2>&1 \|\| true
	echo "--- Copilot output ---"
	cat /tmp/copilot_output.txt \|\| true

	# Extract JSON: find the first {...} block that has a "plans" key
	cat > /tmp/extract_json.py << 'PYEOF'
	import sys, re, json
	text = open('/tmp/copilot_output.txt').read()
	for pat in [r'\{[^{}]"plans"[^{}]\}', r'\{.*?\}']:
	for m in re.findall(pat, text, re.DOTALL):
	try:
	obj = json.loads(m)
	if 'plans' in obj:
	print(json.dumps(obj))
	sys.exit(0)
	except Exception:
	pass
	print('')
	PYEOF
	AI_CONTENT=$(python3 /tmp/extract_json.py 2>/dev/null \|\| true)

	echo "Extracted AI content: $AI_CONTENT"

	if [ -n "$AI_CONTENT" ]; then
	AI_SOURCE="copilot-cli"
	fi

	AI_PLANS=$(echo "$AI_CONTENT" \| jq -r '.plans // empty' 2>/dev/null \|\| true)
	AI_REASON=$(echo "$AI_CONTENT" \| jq -r '.reason // empty' 2>/dev/null \|\| true)

	# Validate: only keep plans that exist on disk
	VALID_PLANS='[]'
	if [ -n "$AI_PLANS" ] && [ "$AI_PLANS" != "null" ]; then
	while IFS= read -r plan; do
	[ -z "$plan" ] && continue
	if [ -f "packages/tests/vscuse/vscode-test-cases/plans/${plan}.json" ]; then
	VALID_PLANS=$(echo "$VALID_PLANS" \| jq --arg p "$plan" '. + [$p]')
	else
	echo "Warning: AI returned unknown plan, skipping: $plan"
	fi
	done < <(echo "$AI_PLANS" \| jq -r '.[]' 2>/dev/null \|\| true)
	fi

	# Force-include impacted plans: a changed test plan must always run, on top
	# of the AI's code-based picks (order-preserving dedup).
	if [ "$IMPACTED_COUNT" -gt 0 ]; then
	VALID_PLANS=$(jq -cn --argjson a "$VALID_PLANS" --argjson b "$IMPACTED_PLANS" \
	'reduce ($a + $b)[] as $p ([]; if index($p) then . else . + [$p] end)')
	if [ -n "$AI_REASON" ]; then
	AI_REASON="${AI_REASON} (Also running ${IMPACTED_COUNT} impacted test plan(s) changed in this PR.)"
	else
	AI_REASON="Running ${IMPACTED_COUNT} impacted test plan(s) changed in this PR."
	fi
	fi

	# Fallback to smoke tests if neither AI nor impacted plans produced anything usable
	IS_FALLBACK="false"
	if [ "$(echo "$VALID_PLANS" \| jq 'length')" -eq 0 ]; then
	echo "No valid AI plans — falling back to smoke test cases."
	VALID_PLANS=$(jq -c '[.smokeTestCases[]]' packages/tests/vscuse/smoking-test-cases.json)
	AI_REASON="AI selection unavailable (source=${AI_SOURCE}); using smoke test cases as fallback."
	IS_FALLBACK="true"
	fi

	PLANS_CSV=$(echo "$VALID_PLANS" \| jq -r 'join(",")')
	echo "test_plans=${PLANS_CSV}" >> "$GITHUB_OUTPUT"
	echo "reason=${AI_REASON}" >> "$GITHUB_OUTPUT"
	echo "ai_source=${AI_SOURCE}" >> "$GITHUB_OUTPUT"
	echo "is_fallback=${IS_FALLBACK}" >> "$GITHUB_OUTPUT"
	echo "Selected: $PLANS_CSV"
	echo "Reason: $AI_REASON"
	# ── 5. Post initial PR comment ─────────────────────────────────────
	- name: Post PR comment with selected test plans
	id: post-comment
	if: steps.code-check.outputs.skip_dispatch != 'true'
	uses: actions/github-script@v6
	env:
	TEST_PLANS: ${{ steps.ai-select.outputs.test_plans }}
	REASON: ${{ steps.ai-select.outputs.reason }}
	AI_SOURCE: ${{ steps.ai-select.outputs.ai_source }}
	IS_FALLBACK: ${{ steps.ai-select.outputs.is_fallback }}
	PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }}
	HEAD_REF: ${{ steps.pr-context.outputs.head_ref }}
	BASE_REF: ${{ steps.pr-context.outputs.base_ref }}
	with:
	script: \|
	const plans = process.env.TEST_PLANS.split(',').filter(Boolean);
	const reason = process.env.REASON \|\| '-';
	const aiSource = process.env.AI_SOURCE \|\| 'unknown';
	const isFallback = process.env.IS_FALLBACK === 'true';
	const prNumber = parseInt(process.env.PR_NUMBER);
	const headRef = process.env.HEAD_REF;
	const baseRef = process.env.BASE_REF;
	const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
	const planList = plans.map(p => `- \`${p}\``).join('\n');

	const header = isFallback
	? `## ⚠️ VscUse Test Plan — Smoke Test Fallback\n\n> AI selection was unavailable (tried: \`${aiSource}\`). Running default smoke tests instead.`
	: `## 🤖 VscUse Test Plan — AI Selected (via \`${aiSource}\`)`;

	const body = [
	header,
	'',
	`Why these tests: ${reason}`,
	'',
	`Branch diff: \`${headRef}\` → \`${baseRef}\``,
	'',
	'Plans queued:',
	planList,
	'',
	'---',
	'\| Step \| Status \|',
	'\|------\|--------\|',
	'\| 1️⃣ Build VSIX (CD) \| ⏳ Running… \|',
	'\| 2️⃣ Build Docker image \| ⌛ Waiting \|',
	'\| 3️⃣ Run UI tests \| ⌛ Waiting \|',
	'',
	`> 🔗 [View pipeline run](${runUrl})`,
	].join('\n');
	const { data: comment } = await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: prNumber,
	body,
	});

	core.setOutput('comment_id', String(comment.id));
	console.log(`Posted comment: ${comment.html_url}`);

	# ── 6. Dispatch dev-smoke-test-pipeline with selected plans ────────
	- name: Dispatch dev-smoke-test-pipeline.yml
	id: dispatch
	if: steps.code-check.outputs.skip_dispatch != 'true'
	uses: actions/github-script@v6
	env:
	TEST_PLANS: ${{ steps.ai-select.outputs.test_plans }}
	HEAD_BRANCH: ${{ steps.pr-context.outputs.head_ref }}
	with:
	github-token: ${{ secrets.GITHUB_TOKEN }}
	script: \|
	const owner = context.repo.owner;
	const repo = context.repo.repo;
	const testPlans = process.env.TEST_PLANS;
	const headBranch = process.env.HEAD_BRANCH;

	// dev-smoke-test-pipeline.yml on dev branch may not have test_plan yet (pending PR merge)
	// Try with test_plan first, fall back to without it if rejected
	let dispatched = false;
	try {
	await github.rest.actions.createWorkflowDispatch({
	owner,
	repo,
	workflow_id: 'dev-smoke-test-pipeline.yml',
	ref: headBranch,
	inputs: {
	test_plan: testPlans,
	email: 'atk-vsuse-test@microsoft.com',
	},
	});
	dispatched = true;
	console.log('Dispatched with test_plan input');
	} catch (err) {
	if (err.status === 422) {
	console.log('test_plan input not supported on target branch, retrying without it...');
	await github.rest.actions.createWorkflowDispatch({
	owner,
	repo,
	workflow_id: 'dev-smoke-test-pipeline.yml',
	ref: headBranch,
	inputs: { email: 'atk-vsuse-test@microsoft.com' },
	});
	dispatched = true;
	console.log('Dispatched without test_plan input');
	} else {
	throw err;
	}
	}

	await new Promise(r => setTimeout(r, 15000));

	const { data: { workflow_runs: runs } } = await github.rest.actions.listWorkflowRuns({
	owner,
	repo,
	workflow_id: 'dev-smoke-test-pipeline.yml',
	branch: headBranch,
	per_page: 5,
	});

	const run = runs.find(r =>
	r.status !== 'completed' \|\|
	(Date.now() - new Date(r.created_at).getTime()) < 60000
	);

	if (!run) throw new Error('Could not locate the dispatched dev-smoke-test-pipeline run');

	core.setOutput('smoke_run_id', String(run.id));
	core.setOutput('smoke_run_url', run.html_url);
	console.log(`Smoke pipeline run ID: ${run.id} URL: ${run.html_url}`);

	# Update comment: pipeline now running
	- name: Update PR comment — pipeline dispatched
	if: steps.code-check.outputs.skip_dispatch != 'true'
	uses: actions/github-script@v6
	env:
	COMMENT_ID: ${{ steps.post-comment.outputs.comment_id }}
	TEST_PLANS: ${{ steps.ai-select.outputs.test_plans }}
	REASON: ${{ steps.ai-select.outputs.reason }}
	RUN_URL: ${{ steps.dispatch.outputs.smoke_run_url }}
	PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }}
	HEAD_REF: ${{ steps.pr-context.outputs.head_ref }}
	BASE_REF: ${{ steps.pr-context.outputs.base_ref }}
	with:
	script: \|
	const commentId = parseInt(process.env.COMMENT_ID);
	const plans = process.env.TEST_PLANS.split(',').filter(Boolean);
	const reason = process.env.REASON \|\| '-';
	const runUrl = process.env.RUN_URL;
	const headRef = process.env.HEAD_REF;
	const baseRef = process.env.BASE_REF;
	const planList = plans.map(p => `- \`${p}\``).join('\n');

	const body = [
	'## 🤖 VscUse Test Plan — AI Selected',
	'',
	`Why these tests: ${reason}`,
	'',
	`Branch diff: \`${headRef}\` → \`${baseRef}\``,
	'',
	'Plans queued:',
	planList,
	'',
	'---',
	'\| Step \| Status \|',
	'\|------\|--------\|',
	'\| 1️⃣ Build VSIX (CD) \| ⏳ Running… \|',
	'\| 2️⃣ Build Docker image \| ⌛ Waiting \|',
	'\| 3️⃣ Run UI tests \| ⌛ Waiting \|',
	'',
	`> 🔗 [View smoke pipeline](${runUrl})`,
	].join('\n');

	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: commentId,
	body,
	});

	# ── 7. Poll smoke pipeline until done ─────────────────────────────
	- name: Wait for smoke pipeline to complete
	id: wait-smoke
	if: steps.code-check.outputs.skip_dispatch != 'true'
	uses: actions/github-script@v6
	env:
	SMOKE_RUN_ID: ${{ steps.dispatch.outputs.smoke_run_id }}
	with:
	github-token: ${{ secrets.GITHUB_TOKEN }}
	script: \|
	const owner = context.repo.owner;
	const repo = context.repo.repo;
	const runId = parseInt(process.env.SMOKE_RUN_ID);
	const start = Date.now();
	const EIGHT_HOURS = 8 * 60 * 60 * 1000;

	// Try to extract the actual UI-test run URL from the smoke pipeline's
	// `run-ui-test` job logs. That job dispatches ui-test-vscuse-template.yml
	// and prints "UI Test run ID: <id>" — surfacing it lets the PR comment
	// link straight to the real test report instead of the smoke wrapper.
	const extractUiTestUrl = async () => {
	try {
	const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
	owner, repo, run_id: runId, per_page: 100,
	});
	const uiJob = jobs.find(j => j.name === 'run-ui-test');
	if (!uiJob) {
	console.log('run-ui-test job not found on smoke pipeline.');
	return '';
	}
	const logs = await github.request(
	'GET /repos/{owner}/{repo}/actions/jobs/{job_id}/logs',
	{ owner, repo, job_id: uiJob.id }
	);
	const match = String(logs.data \|\| '').match(/UI Test run ID:\s*(\d+)/);
	if (!match) {
	console.log('Could not find "UI Test run ID:" line in run-ui-test logs.');
	return '';
	}
	const uiRunId = match[1];
	const uiUrl = `https://github.com/${owner}/${repo}/actions/runs/${uiRunId}`;
	console.log(`Extracted UI test run: ${uiUrl}`);
	return uiUrl;
	} catch (err) {
	console.log(`Failed to extract UI test run URL: ${err.message}`);
	return '';
	}
	};

	// On the UI test run, the `report` job (ui-test-vscuse-common.yml) uploads
	// the aggregated email-body.html to Azure Blob and prints
	// "Report Summary URL: <url>". The PR-trigger path emails the report to an
	// unattended mailbox, so this single aggregate blob link is the only
	// user-facing report surface.
	const extractReportUrl = async (uiUrl) => {
	const idMatch = (uiUrl \|\| '').match(/\/runs\/(\d+)/);
	if (!idMatch) return '';
	try {
	// Paginate: a UI-test run has one matrix job per selected test plan and
	// can exceed 100 jobs. The report job is near the end, so a single
	// per_page=100 page would miss it and drop the report link.
	const uiJobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
	owner, repo, run_id: parseInt(idMatch[1]), per_page: 100,
	});
	// The report job is invoked via a reusable workflow, so its rendered
	// name is typically "<caller_job> / report". Match the trailing token.
	const reportJob = uiJobs.find(j =>
	/(^\|\/\s*)report$/.test((j.name \|\| '').trim())
	);
	if (!reportJob) { console.log('report job not found on UI test run.'); return ''; }
	const log = await github.request(
	'GET /repos/{owner}/{repo}/actions/jobs/{job_id}/logs',
	{ owner, repo, job_id: reportJob.id }
	);
	const m = String(log.data \|\| '').match(/Report Summary URL:\s*(https?:\/\/\S+)/);
	if (!m) { console.log('No "Report Summary URL:" marker found in report job logs.'); return ''; }
	console.log(`Extracted aggregated report URL: ${m[1]}`);
	return m[1];
	} catch (err) {
	console.log(`Failed to extract aggregated report URL: ${err.message}`);
	return '';
	}
	};

	// Count per-plan (matrix) job outcomes on the UI test run. Each selected
	// plan renders as a "<caller_job> / Case-<plan>" job (see ui-test-vscuse-common.yml
	// job `main`, name "Case-${{ matrix.test_plan }}"). Retries re-run failed jobs as
	// new run attempts, so list with filter=all and keep the latest attempt per plan
	// to reflect each plan's final pass/fail status.
	const extractTestCounts = async (uiUrl) => {
	const idMatch = (uiUrl \|\| '').match(/\/runs\/(\d+)/);
	if (!idMatch) return null;
	try {
	const uiJobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
	owner, repo, run_id: parseInt(idMatch[1]), filter: 'all', per_page: 100,
	});
	// Keep the newest attempt for each distinct Case-<plan> job.
	const latestByName = new Map();
	for (const j of uiJobs) {
	const name = (j.name \|\| '').trim();
	if (!/(?:^\|\/\s*)Case-[^/]+$/.test(name)) continue;
	const prev = latestByName.get(name);
	if (!prev \|\| (j.run_attempt \|\| 0) >= (prev.run_attempt \|\| 0)) {
	latestByName.set(name, j);
	}
	}
	let passed = 0, failed = 0;
	for (const j of latestByName.values()) {
	if (j.conclusion === 'success') passed++;
	else if (j.conclusion === 'skipped' \|\| j.conclusion == null) continue;
	else failed++;
	}
	const total = passed + failed;
	console.log(`Test-plan results: ${passed} passed, ${failed} failed (of ${total} plans).`);
	return { passed, failed, total };
	} catch (err) {
	console.log(`Failed to compute test-plan counts: ${err.message}`);
	return null;
	}
	};

	while (Date.now() - start < EIGHT_HOURS) {
	const { data: run } = await github.rest.actions.getWorkflowRun({ owner, repo, run_id: runId });
	console.log(`Smoke pipeline: ${run.status} / ${run.conclusion}`);
	if (run.status === 'completed') {
	core.setOutput('conclusion', run.conclusion);
	core.setOutput('run_url', run.html_url);
	const uiUrl = await extractUiTestUrl();
	core.setOutput('ui_test_url', uiUrl);
	core.setOutput('report_url', await extractReportUrl(uiUrl));
	const counts = await extractTestCounts(uiUrl);
	core.setOutput('passed_count', counts ? String(counts.passed) : '');
	core.setOutput('failed_count', counts ? String(counts.failed) : '');
	core.setOutput('total_count', counts ? String(counts.total) : '');
	return;
	}
	await new Promise(r => setTimeout(r, 60000));
	}
	core.setOutput('conclusion', 'timed_out');
	const uiUrl = await extractUiTestUrl();
	core.setOutput('ui_test_url', uiUrl);
	core.setOutput('report_url', await extractReportUrl(uiUrl));
	const counts = await extractTestCounts(uiUrl);
	core.setOutput('passed_count', counts ? String(counts.passed) : '');
	core.setOutput('failed_count', counts ? String(counts.failed) : '');
	core.setOutput('total_count', counts ? String(counts.total) : '');
	console.log('Smoke pipeline timed out after 8 hours — reporting as timed_out (workflow stays green; this run is informational only).');

	# ── 8. Resolve PR comment with final result ────────────────────────
	- name: Resolve PR comment with test results
	if: always() && steps.code-check.outputs.skip_dispatch != 'true'
	uses: actions/github-script@v6
	env:
	COMMENT_ID: ${{ steps.post-comment.outputs.comment_id }}
	TEST_PLANS: ${{ steps.ai-select.outputs.test_plans }}
	REASON: ${{ steps.ai-select.outputs.reason }}
	CONCLUSION: ${{ steps.wait-smoke.outputs.conclusion }}
	SMOKE_URL: ${{ steps.wait-smoke.outputs.run_url }}
	UI_TEST_URL: ${{ steps.wait-smoke.outputs.ui_test_url }}
	REPORT_URL: ${{ steps.wait-smoke.outputs.report_url }}
	PASSED_COUNT: ${{ steps.wait-smoke.outputs.passed_count }}
	FAILED_COUNT: ${{ steps.wait-smoke.outputs.failed_count }}
	TOTAL_COUNT: ${{ steps.wait-smoke.outputs.total_count }}
	PR_NUMBER: ${{ steps.pr-context.outputs.pr_number }}
	HEAD_REF: ${{ steps.pr-context.outputs.head_ref }}
	BASE_REF: ${{ steps.pr-context.outputs.base_ref }}
	with:
	script: \|
	const commentId = parseInt(process.env.COMMENT_ID);
	if (!commentId) { console.log('No comment ID — skipping update'); return; }

	const plans = (process.env.TEST_PLANS \|\| '').split(',').filter(Boolean);
	const reason = process.env.REASON \|\| '-';
	const conclusion = process.env.CONCLUSION \|\| 'unknown';
	const smokeUrl = process.env.SMOKE_URL \|\| '';
	const uiTestUrl = process.env.UI_TEST_URL \|\| '';
	const reportUrl = process.env.REPORT_URL \|\| '';
	const headRef = process.env.HEAD_REF;
	const baseRef = process.env.BASE_REF;
	const planList = plans.map(p => `- \`${p}\``).join('\n');

	const passedCount = process.env.PASSED_COUNT \|\| '';
	const failedCount = process.env.FAILED_COUNT \|\| '';
	const totalCount = process.env.TOTAL_COUNT \|\| '';
	const hasCounts = passedCount !== '' && failedCount !== '' && Number(totalCount) > 0;
	const resultsLine = hasCounts
	? `Results: ✅ ${passedCount} passed · ❌ ${failedCount} failed (of ${totalCount} plans)`
	: '';

	const succeeded = conclusion === 'success';
	const icon = succeeded ? '✅' : '❌';
	const statusText = succeeded ? 'All tests passed' : `Tests ${conclusion}`;
	const stepThreeText = hasCounts
	? `${icon} ${statusText} (${passedCount}/${totalCount} passed)`
	: `${icon} ${statusText}`;

	const body = [
	`## ${icon} VscUse Test Plan — ${statusText}`,
	'',
	`Why these tests: ${reason}`,
	'',
	`Branch diff: \`${headRef}\` → \`${baseRef}\``,
	'',
	resultsLine,
	'',
	'Plans run:',
	planList,
	'',
	'---',
	'\| Step \| Status \|',
	'\|------\|--------\|',
	'\| 1️⃣ Build VSIX (CD) \| ✅ Done \|',
	'\| 2️⃣ Build Docker image \| ✅ Done \|',
	`\| 3️⃣ Run UI tests \| ${stepThreeText} \|`,
	'',
	uiTestUrl ? `> 🎯 [Actual UI test run](${uiTestUrl})` : '',
	smokeUrl ? `> 🔗 [Full pipeline results](${smokeUrl})` : '',
	reportUrl ? `> 📊 [Detailed test report](${reportUrl})` : '',
	'',
	'<details>',
	'<summary>ℹ️ How were these tests selected?</summary>',
	'',
	'GitHub Copilot (Claude Sonnet 4.6, high reasoning) analysed the PR title, description, and the diff between',
	`\`${headRef}\` and \`${baseRef}\``,
	`to pick the most relevant test plans from [\`packages/tests/vscuse/Index.md\`](../blob/${baseRef}/packages/tests/vscuse/Index.md).`,
	'',
	'</details>',
	].join('\n');

	await github.rest.issues.updateComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	comment_id: commentId,
	body,
	});

	console.log(`Resolved comment ${commentId}: ${conclusion}`);
	// Informational workflow: never fail the job, even on test failure or timeout.
	// The PR comment carries the result; merge gating must not depend on this run.
	if (!succeeded) {
	console.log(`VscUse tests concluded: ${conclusion}. See: ${smokeUrl}`);
	}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

refactor: update click input box description #771

Workflow file

refactor: update click input box description #771

Uh oh!

Workflow file for this run