Skill Quality Report — Nightly Scan #8

Workflow file for this run

.github/workflows/skill-quality-report.yml at e252678

	name: Skill Quality Report — Nightly Scan

	on:
	schedule:
	- cron: "0 3 * * *" # 3:00 AM UTC daily
	workflow_dispatch: # allow manual trigger

	permissions:
	contents: read
	discussions: write
	issues: write # fallback if Discussions are not enabled

	jobs:
	nightly-scan:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
	with:
	fetch-depth: 0 # full history for git-log author fallback

	# ── Download & cache skill-validator ──────────────────────────
	- name: Get cache key date
	id: cache-date
	run: echo "date=$(date +%Y-%m-%d)" >> "$GITHUB_OUTPUT"

	- name: Restore skill-validator from cache
	id: cache-sv
	uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
	with:
	path: .skill-validator
	key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}
	restore-keys: \|
	skill-validator-linux-x64-

	- name: Download skill-validator
	if: steps.cache-sv.outputs.cache-hit != 'true'
	run: \|
	mkdir -p .skill-validator
	curl -fsSL \
	"https://github.com/dotnet/skills/releases/download/skill-validator-nightly/skill-validator-linux-x64.tar.gz" \
	-o .skill-validator/skill-validator-linux-x64.tar.gz
	tar -xzf .skill-validator/skill-validator-linux-x64.tar.gz -C .skill-validator
	rm .skill-validator/skill-validator-linux-x64.tar.gz
	chmod +x .skill-validator/skill-validator

	- name: Save skill-validator to cache
	if: steps.cache-sv.outputs.cache-hit != 'true'
	uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
	with:
	path: .skill-validator
	key: skill-validator-linux-x64-${{ steps.cache-date.outputs.date }}

	# ── Run full scan ─────────────────────────────────────────────
	- name: Run skill-validator check on all skills
	id: check-skills
	run: \|
	set +e
	set -o pipefail
	.skill-validator/skill-validator check \
	--skills ./skills \
	--verbose \
	2>&1 \| tee sv-skills-output.txt
	echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
	set +o pipefail
	set -e

	- name: Run skill-validator check on all agents
	id: check-agents
	run: \|
	set +e
	set -o pipefail
	AGENT_FILES=$(find agents -name '*.agent.md' -type f 2>/dev/null \| tr '\n' ' ')
	if [ -n "$AGENT_FILES" ]; then
	.skill-validator/skill-validator check \
	--agents $AGENT_FILES \
	--verbose \
	2>&1 \| tee sv-agents-output.txt
	echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
	else
	echo "No agent files found."
	echo "" > sv-agents-output.txt
	echo "exit_code=0" >> "$GITHUB_OUTPUT"
	fi
	set +o pipefail
	set -e

	# ── Build report with author attribution ──────────────────────
	- name: Build quality report
	id: report
	uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
	with:
	script: \|
	const fs = require('fs');
	const path = require('path');
	const { execSync } = require('child_process');

	// ── Parse CODEOWNERS ──────────────────────────────────
	function parseCodeowners() {
	const map = new Map();
	try {
	const raw = fs.readFileSync('CODEOWNERS', 'utf8');
	for (const line of raw.split('\n')) {
	const trimmed = line.trim();
	if (!trimmed \|\| trimmed.startsWith('#')) continue;
	const parts = trimmed.split(/\s+/);
	if (parts.length >= 2) {
	const filePath = parts[0].replace(/^\//, '').replace(/\/$/, '');
	const owners = parts.slice(1).filter(p => p.startsWith('@'));
	if (owners.length > 0) {
	map.set(filePath, owners);
	}
	}
	}
	} catch (e) {
	console.log('Could not parse CODEOWNERS:', e.message);
	}
	return map;
	}

	// ── Resolve author for a path ─────────────────────────
	function resolveAuthor(resourcePath, codeowners) {
	// CODEOWNERS semantics: last matching rule wins.
	// Also treat "*" as a match-all default rule.
	let matchedOwners = null;
	for (const [pattern, owners] of codeowners) {
	if (
	pattern === '*' \|\|
	resourcePath === pattern \|\|
	resourcePath.startsWith(pattern + '/')
	) {
	matchedOwners = owners;
	}
	}
	if (matchedOwners && matchedOwners.length > 0) {
	return matchedOwners.join(', ');
	}
	// Fallback: git log
	try {
	const author = execSync(
	`git log --format='%aN' --follow -1 -- "${resourcePath}"`,
	{ encoding: 'utf8' }
	).trim();
	return author \|\| 'unknown';
	} catch {
	return 'unknown';
	}
	}

	// ── Parse skill-validator output ──────────────────────
	// The output is a text report; we preserve it as-is and
	// augment it with author info in the summary.
	const skillsOutput = fs.readFileSync('sv-skills-output.txt', 'utf8').trim();
	const agentsOutput = fs.existsSync('sv-agents-output.txt')
	? fs.readFileSync('sv-agents-output.txt', 'utf8').trim()
	: '';

	const codeowners = parseCodeowners();

	// Count findings
	// The skill-validator uses emoji markers: ❌ for errors, ⚠ for warnings, ℹ for advisories
	const combined = skillsOutput + '\n' + agentsOutput;
	const errorCount = (combined.match(/❌/g) \|\| []).length;
	const warningCount = (combined.match(/⚠/g) \|\| []).length;
	const advisoryCount = (combined.match(/ℹ\uFE0F?/g) \|\| []).length;

	// Count total skills & agents checked
	let skillDirs = [];
	try {
	skillDirs = fs.readdirSync('skills', { withFileTypes: true })
	.filter(d => d.isDirectory())
	.map(d => d.name);
	} catch {}

	let agentFiles = [];
	try {
	agentFiles = fs.readdirSync('agents')
	.filter(f => f.endsWith('.agent.md'));
	} catch {}

	// ── Build author-attributed summary ───────────────────
	// Extract per-resource blocks from output. The validator
	// prints skill names as headers — we annotate them with
	// the resolved owner.
	function annotateWithAuthors(output, kind) {
	if (!output) return '_No findings._';
	const lines = output.split('\n');
	const annotated = [];
	for (const line of lines) {
	// Skill names appear as headers, e.g. "## skill-name" or "skill-name:"
	const headerMatch = line.match(/^(?:#{1,3}\s+)?([a-z0-9][a-z0-9-]+(?:\.[a-z0-9.-]+)?)\b/);
	if (headerMatch) {
	const name = headerMatch[1];
	const resourcePath = kind === 'skill'
	? `skills/${name}`
	: `agents/${name}.agent.md`;
	const author = resolveAuthor(resourcePath, codeowners);
	annotated.push(`${line} — ${author}`);
	} else {
	annotated.push(line);
	}
	}
	return annotated.join('\n');
	}

	const today = new Date().toISOString().split('T')[0];

	const title = `Skill Quality Report — ${today}`;

	const annotatedSkills = annotateWithAuthors(skillsOutput, 'skill');
	const annotatedAgents = annotateWithAuthors(agentsOutput, 'agent');

	// ── Body size management ──────────────────────────────
	// GitHub body limit is ~65536 UTF-8 bytes for both
	// Discussions and Issues. When the full report fits, we
	// inline everything. When it doesn't, the body gets a
	// compact summary and the verbose sections are written to
	// separate files that get posted as follow-up comments.
	const MAX_BYTES = 65000; // leave margin

	function makeDetailsBlock(heading, summary, content) {
	return [
	`## ${heading}`, '',
	'<details>',
	`<summary>${summary}</summary>`, '',
	'```', content, '```', '',
	'</details>',
	].join('\n');
	}

	const summaryLines = [
	`# ${title}`, '',
	`${skillDirs.length} skills and ${agentFiles.length} agents scanned.`, '',
	'\| Severity \| Count \|',
	'\|----------\|-------\|',
	`\| ⛔ Errors \| ${errorCount} \|`,
	`\| ⚠️ Warnings \| ${warningCount} \|`,
	`\| ℹ️ Advisories \| ${advisoryCount} \|`, '',
	'---',
	];
	const footer = `\n---\n\n_Generated by the [Skill Validator nightly scan](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/skill-quality-report.yml)._`;

	const skillsBlock = makeDetailsBlock('Skills', 'Full skill-validator output for skills', annotatedSkills);
	const agentsBlock = makeDetailsBlock('Agents', 'Full skill-validator output for agents', annotatedAgents);

	// Try full inline body first
	const fullBody = summaryLines.join('\n') + '\n\n' + skillsBlock + '\n\n' + agentsBlock + footer;

	const commentParts = []; // overflow comment files

	let finalBody;
	if (Buffer.byteLength(fullBody, 'utf8') <= MAX_BYTES) {
	finalBody = fullBody;
	} else {
	// Details won't fit inline — move them to follow-up comments
	const bodyNote = '\n\n> Note: Detailed output is posted in the comments below (too large for the discussion body).\n';
	finalBody = summaryLines.join('\n') + bodyNote + footer;

	// Split each section into ≤65 KB chunks
	function chunkContent(label, content) {
	const prefix = `## ${label}\n\n\`\`\`\n`;
	const suffix = '\n```';
	const overhead = Buffer.byteLength(prefix + suffix, 'utf8');
	const budget = MAX_BYTES - overhead;

	const buf = Buffer.from(content, 'utf8');
	if (buf.length <= budget) {
	return [prefix + content + suffix];
	}
	const parts = [];
	let offset = 0;
	let partNum = 1;
	while (offset < buf.length) {
	const slice = buf.slice(offset, offset + budget).toString('utf8');
	// Remove trailing replacement char from mid-codepoint cut
	const clean = slice.replace(/\uFFFD$/, '');
	const hdr = `## ${label} (part ${partNum})\n\n\`\`\`\n`;
	parts.push(hdr + clean + suffix);
	offset += Buffer.byteLength(clean, 'utf8');
	partNum++;
	}
	return parts;
	}

	commentParts.push(...chunkContent('Skills', annotatedSkills));
	commentParts.push(...chunkContent('Agents', annotatedAgents));
	}

	core.setOutput('title', title);
	core.setOutput('body_file', 'report-body.md');

	fs.writeFileSync('report-body.md', finalBody);

	// Write overflow comment parts as numbered files
	for (let i = 0; i < commentParts.length; i++) {
	fs.writeFileSync(`report-comment-${i}.md`, commentParts[i]);
	}
	core.setOutput('comment_count', String(commentParts.length));

	# ── Create Discussion (preferred) or Issue (fallback) ────────
	- name: Create Discussion
	id: create-discussion
	continue-on-error: true
	uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
	with:
	script: \|
	const fs = require('fs');
	const title = '${{ steps.report.outputs.title }}'.replace(/'/g, "\\'");
	const body = fs.readFileSync('report-body.md', 'utf8');
	const commentCount = parseInt('${{ steps.report.outputs.comment_count }}' \|\| '0', 10);

	// Find the "Skill Quality Reports" category
	const categoriesResult = await github.graphql(`
	query($owner: String!, $repo: String!) {
	repository(owner: $owner, name: $repo) {
	id
	discussionCategories(first: 25) {
	nodes { id name }
	}
	}
	}
	`, {
	owner: context.repo.owner,
	repo: context.repo.repo,
	});

	const repo = categoriesResult.repository;
	const categories = repo.discussionCategories.nodes;
	const category = categories.find(c =>
	c.name === 'Skill Quality Reports'
	);

	if (!category) {
	core.setFailed('Discussion category "Skill Quality Reports" not found. Falling back to issue.');
	return;
	}

	const result = await github.graphql(`
	mutation($repoId: ID!, $categoryId: ID!, $title: String!, $body: String!) {
	createDiscussion(input: {
	repositoryId: $repoId,
	categoryId: $categoryId,
	title: $title,
	body: $body
	}) {
	discussion { id url }
	}
	}
	`, {
	repoId: repo.id,
	categoryId: category.id,
	title: title,
	body: body,
	});

	const discussionId = result.createDiscussion.discussion.id;
	console.log(`Discussion created: ${result.createDiscussion.discussion.url}`);

	// Post overflow detail comments
	for (let i = 0; i < commentCount; i++) {
	const commentBody = fs.readFileSync(`report-comment-${i}.md`, 'utf8');
	await github.graphql(`
	mutation($discussionId: ID!, $body: String!) {
	addDiscussionComment(input: {
	discussionId: $discussionId,
	body: $body
	}) {
	comment { id }
	}
	}
	`, { discussionId, body: commentBody });
	console.log(`Posted detail comment ${i + 1}/${commentCount}`);
	}

	- name: Fallback — Create Issue
	if: steps.create-discussion.outcome == 'failure'
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	# Create label if it doesn't exist (ignore errors if it already exists)
	gh label create "skill-quality" --description "Automated skill quality reports" --color "d4c5f9" 2>/dev/null \|\| true
	ISSUE_URL=$(gh issue create \
	--title "${{ steps.report.outputs.title }}" \
	--body-file report-body.md \
	--label "skill-quality")
	echo "Created issue: $ISSUE_URL"

	# Post overflow detail comments on the issue
	COMMENT_COUNT=${{ steps.report.outputs.comment_count }}
	for i in $(seq 0 $(( ${COMMENT_COUNT:-0} - 1 ))); do
	if [ -f "report-comment-${i}.md" ]; then
	gh issue comment "$ISSUE_URL" --body-file "report-comment-${i}.md"
	echo "Posted detail comment $((i+1))/${COMMENT_COUNT}"
	fi
	done

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Skill Quality Report — Nightly Scan #8

Workflow file

Skill Quality Report — Nightly Scan #8

Uh oh!

Workflow file for this run