|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Copyright (c) Microsoft Corporation. All rights reserved. |
| 3 | +# Licensed under the MIT License. |
| 4 | +# |
| 5 | +# Knowledge accuracy audit using GitHub Models API. |
| 6 | +# Compares each knowledge file against the actual source code it documents. |
| 7 | +# Detects: inaccuracies, stale descriptions, deleted/renamed files, |
| 8 | +# new untracked source files, and refactored APIs. |
| 9 | +# |
| 10 | +# Usage: knowledge-accuracy.sh <repo> |
| 11 | +# Requires: GITHUB_TOKEN env var, jq, gh CLI |
| 12 | + |
| 13 | +set -euo pipefail |
| 14 | + |
| 15 | +REPO="$1" |
| 16 | +COMMIT_SHA=$(git rev-parse HEAD) |
| 17 | +MODEL="openai/gpt-4o-mini" |
| 18 | +API_URL="https://models.github.ai/inference/chat/completions" |
| 19 | + |
| 20 | +echo "=== Knowledge Accuracy Audit ===" |
| 21 | +echo "Repository: ${REPO}" |
| 22 | +echo "Commit: ${COMMIT_SHA}" |
| 23 | + |
| 24 | +# ── Phase 1: Inventory ────────────────────────────────────────────────── |
| 25 | +# Build a map of knowledge files → referenced source files, and discover |
| 26 | +# source files not covered by any knowledge doc. |
| 27 | + |
| 28 | +echo "" |
| 29 | +echo "=== Phase 1: Inventory ===" |
| 30 | + |
| 31 | +ALL_FINDINGS="[]" |
| 32 | +KNOWLEDGE_DIR="docs/knowledge" |
| 33 | +KNOWLEDGE_FILES=() |
| 34 | +for kf in "${KNOWLEDGE_DIR}"/*.md; do |
| 35 | + [ -f "$kf" ] && KNOWLEDGE_FILES+=("$kf") |
| 36 | +done |
| 37 | +echo "Knowledge files: ${#KNOWLEDGE_FILES[@]}" |
| 38 | + |
| 39 | +# Build full source inventory for coverage analysis |
| 40 | +SOURCE_FILES=$(find src/ bindings/ -type f -name '*.rs' 2>/dev/null | sort) |
| 41 | +SOURCE_COUNT=$(echo "$SOURCE_FILES" | wc -l | tr -d ' ') |
| 42 | +echo "Source files in repo: ${SOURCE_COUNT}" |
| 43 | + |
| 44 | +# Track which source files are referenced by at least one knowledge doc |
| 45 | +REFERENCED_FILES="" |
| 46 | + |
| 47 | +# ── Phase 2: Per-Knowledge-File Analysis ───────────────────────────────── |
| 48 | + |
| 49 | +echo "" |
| 50 | +echo "=== Phase 2: Knowledge File Analysis ===" |
| 51 | + |
| 52 | +for kf in "${KNOWLEDGE_FILES[@]}"; do |
| 53 | + kf_name=$(basename "$kf") |
| 54 | + echo "" |
| 55 | + echo "--- Checking: ${kf_name} ---" |
| 56 | + |
| 57 | + # Extract source file references from this knowledge doc |
| 58 | + REFS=$(grep -oE '(src|bindings|tests|examples|xtask)/[a-zA-Z0-9_/.*-]+\.(rs|toml|yml|yaml)' "$kf" 2>/dev/null | sort -u || true) |
| 59 | + # Also extract backtick-quoted module paths like `src/value.rs` |
| 60 | + REFS2=$(grep -oE '`(src|bindings|tests|examples|xtask)/[a-zA-Z0-9_/.*-]+\.(rs|toml)`' "$kf" 2>/dev/null | tr -d '`' | sort -u || true) |
| 61 | + ALL_REFS=$(printf '%s\n%s' "$REFS" "$REFS2" | sort -u | grep -v '^$' || true) |
| 62 | + |
| 63 | + REF_COUNT=$(echo "$ALL_REFS" | grep -c . || echo 0) |
| 64 | + echo " Referenced files: ${REF_COUNT}" |
| 65 | + |
| 66 | + # Check for deleted/missing files |
| 67 | + MISSING="" |
| 68 | + EXISTING_REFS="" |
| 69 | + while IFS= read -r ref; do |
| 70 | + [ -z "$ref" ] && continue |
| 71 | + # Handle glob patterns (e.g., src/builtins/*.rs) |
| 72 | + if [[ "$ref" == *"*"* ]]; then |
| 73 | + expanded=$(ls $ref 2>/dev/null | head -5 || true) |
| 74 | + if [ -z "$expanded" ]; then |
| 75 | + MISSING="${MISSING} - ${ref} (glob pattern matches nothing)\n" |
| 76 | + else |
| 77 | + EXISTING_REFS="${EXISTING_REFS}${expanded}\n" |
| 78 | + fi |
| 79 | + elif [ ! -f "$ref" ] && [ ! -d "$ref" ]; then |
| 80 | + MISSING="${MISSING} - ${ref}\n" |
| 81 | + else |
| 82 | + EXISTING_REFS="${EXISTING_REFS}${ref}\n" |
| 83 | + REFERENCED_FILES="${REFERENCED_FILES}${ref}\n" |
| 84 | + fi |
| 85 | + done <<< "$ALL_REFS" |
| 86 | + |
| 87 | + if [ -n "$MISSING" ]; then |
| 88 | + echo " ⚠ Missing/deleted files:" |
| 89 | + echo -e "$MISSING" | sed 's/^/ /' |
| 90 | + # Add as a finding |
| 91 | + missing_text=$(echo -e "$MISSING" | sed '/^$/d') |
| 92 | + ALL_FINDINGS=$(echo "$ALL_FINDINGS" | jq -c --arg kf "$kf_name" --arg missing "$missing_text" \ |
| 93 | + '. + [{ |
| 94 | + "severity": "important", |
| 95 | + "perspective": "knowledge-accuracy", |
| 96 | + "title": ("Missing source files referenced by " + $kf), |
| 97 | + "file": ("docs/knowledge/" + $kf), |
| 98 | + "snippet": "", |
| 99 | + "body": ("The following files are referenced but no longer exist. They may have been deleted or renamed:\n" + $missing + "\n\nUpdate the knowledge file to reflect current file locations.") |
| 100 | + }]') |
| 101 | + fi |
| 102 | + |
| 103 | + # Read the knowledge file content |
| 104 | + KF_CONTENT=$(cat "$kf") |
| 105 | + |
| 106 | + # Read existing referenced source files (truncate each to 6KB) |
| 107 | + SOURCE_CONTEXT="" |
| 108 | + while IFS= read -r ref; do |
| 109 | + [ -z "$ref" ] && continue |
| 110 | + if [ -f "$ref" ]; then |
| 111 | + SOURCE_CONTEXT="${SOURCE_CONTEXT} |
| 112 | +--- source: ${ref} --- |
| 113 | +$(head -c 6000 "$ref") |
| 114 | +" |
| 115 | + fi |
| 116 | + done <<< "$(echo -e "$EXISTING_REFS" | sort -u)" |
| 117 | + |
| 118 | + # Skip LLM call if no source context |
| 119 | + if [ -z "$SOURCE_CONTEXT" ]; then |
| 120 | + echo " No source files to compare against, skipping LLM check" |
| 121 | + continue |
| 122 | + fi |
| 123 | + |
| 124 | + # Call LLM to compare knowledge doc against actual source |
| 125 | + echo " Comparing against source code..." |
| 126 | + |
| 127 | + PROMPT="You are auditing a knowledge documentation file for accuracy against the actual source code. |
| 128 | +
|
| 129 | +KNOWLEDGE FILE: ${kf_name} |
| 130 | +${KF_CONTENT} |
| 131 | +
|
| 132 | +ACTUAL SOURCE CODE: |
| 133 | +${SOURCE_CONTEXT} |
| 134 | +
|
| 135 | +=== TASK === |
| 136 | +Compare the knowledge file against the actual source code. Find: |
| 137 | +
|
| 138 | +1. **INACCURACIES**: Statements in the knowledge doc that contradict the actual code |
| 139 | + (wrong types, wrong function signatures, wrong behavior descriptions, wrong module structure) |
| 140 | +
|
| 141 | +2. **STALE CONTENT**: Descriptions of features, APIs, or patterns that have been |
| 142 | + refactored, renamed, or significantly changed in the source |
| 143 | +
|
| 144 | +3. **MISSING COVERAGE**: Important public APIs, types, or patterns in the source code |
| 145 | + that the knowledge doc should mention but doesn't |
| 146 | +
|
| 147 | +Focus on factual errors that would mislead someone using this knowledge doc. |
| 148 | +Do NOT report style issues or minor wording preferences. |
| 149 | +
|
| 150 | +=== OUTPUT FORMAT === |
| 151 | +Return a JSON array. Each finding: |
| 152 | +- \"severity\": \"critical\" (factual error) | \"important\" (stale/missing) | \"suggestion\" (minor gap) |
| 153 | +- \"title\": one-sentence heading |
| 154 | +- \"file\": \"docs/knowledge/${kf_name}\" |
| 155 | +- \"snippet\": the specific incorrect or stale text from the knowledge doc (quote it exactly) |
| 156 | +- \"body\": explanation of what's wrong and what the correct information is, citing the actual source code |
| 157 | +
|
| 158 | +If the knowledge file is accurate, return: [] |
| 159 | +Return ONLY valid JSON." |
| 160 | + |
| 161 | + RESPONSE=$(jq -n \ |
| 162 | + --arg model "$MODEL" \ |
| 163 | + --arg prompt "$PROMPT" \ |
| 164 | + '{ |
| 165 | + model: $model, |
| 166 | + messages: [ |
| 167 | + {role: "system", content: "You are a documentation accuracy auditor. Return findings as a JSON array only."}, |
| 168 | + {role: "user", content: $prompt} |
| 169 | + ], |
| 170 | + temperature: 0.1 |
| 171 | + }' | curl -s -X POST "$API_URL" \ |
| 172 | + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ |
| 173 | + -H "Content-Type: application/json" \ |
| 174 | + -d @- 2>/dev/null || echo '{"error": "API call failed"}') |
| 175 | + |
| 176 | + CONTENT=$(echo "$RESPONSE" | jq -r '.choices[0].message.content // empty' 2>/dev/null || true) |
| 177 | + |
| 178 | + if [ -z "$CONTENT" ]; then |
| 179 | + ERROR_MSG=$(echo "$RESPONSE" | jq -r '.error // .message // "Unknown error"' 2>/dev/null || echo "Unknown error") |
| 180 | + echo " API error: ${ERROR_MSG}" |
| 181 | + continue |
| 182 | + fi |
| 183 | + |
| 184 | + CONTENT=$(echo "$CONTENT" | sed 's/^```json//; s/^```//; /^$/d') |
| 185 | + |
| 186 | + if ! echo "$CONTENT" | jq -e 'type == "array"' > /dev/null 2>&1; then |
| 187 | + echo " Invalid JSON response, skipping" |
| 188 | + continue |
| 189 | + fi |
| 190 | + |
| 191 | + # Tag each finding with the perspective |
| 192 | + TAGGED=$(echo "$CONTENT" | jq -c '[.[] | . + {perspective: "knowledge-accuracy"}]') |
| 193 | + COUNT=$(echo "$TAGGED" | jq 'length') |
| 194 | + echo " Findings: ${COUNT}" |
| 195 | + |
| 196 | + ALL_FINDINGS=$(echo "$ALL_FINDINGS" "$TAGGED" | jq -s 'add') |
| 197 | +done |
| 198 | + |
| 199 | +# ── Phase 3: Coverage Analysis ─────────────────────────────────────────── |
| 200 | +# Find important source files not covered by any knowledge doc. |
| 201 | + |
| 202 | +echo "" |
| 203 | +echo "=== Phase 3: Coverage Analysis ===" |
| 204 | + |
| 205 | +REFERENCED_UNIQUE=$(echo -e "$REFERENCED_FILES" | sort -u | grep -v '^$' || true) |
| 206 | + |
| 207 | +# Find source files with significant content (>100 lines) not referenced |
| 208 | +UNCOVERED="" |
| 209 | +UNCOVERED_COUNT=0 |
| 210 | +while IFS= read -r src; do |
| 211 | + [ -z "$src" ] && continue |
| 212 | + if ! echo "$REFERENCED_UNIQUE" | grep -qF "$src"; then |
| 213 | + line_count=$(wc -l < "$src" 2>/dev/null | tr -d ' ') |
| 214 | + if [ "$line_count" -gt 100 ]; then |
| 215 | + UNCOVERED="${UNCOVERED} - ${src} (${line_count} lines)\n" |
| 216 | + UNCOVERED_COUNT=$((UNCOVERED_COUNT + 1)) |
| 217 | + fi |
| 218 | + fi |
| 219 | +done <<< "$SOURCE_FILES" |
| 220 | + |
| 221 | +echo "Significant uncovered source files: ${UNCOVERED_COUNT}" |
| 222 | + |
| 223 | +if [ "$UNCOVERED_COUNT" -gt 0 ]; then |
| 224 | + uncovered_text=$(echo -e "$UNCOVERED" | sed '/^$/d' | head -20) |
| 225 | + ALL_FINDINGS=$(echo "$ALL_FINDINGS" | jq -c --arg files "$uncovered_text" --arg count "$UNCOVERED_COUNT" \ |
| 226 | + '. + [{ |
| 227 | + "severity": "suggestion", |
| 228 | + "perspective": "knowledge-accuracy", |
| 229 | + "title": ($count + " significant source files have no knowledge documentation"), |
| 230 | + "file": "docs/knowledge/", |
| 231 | + "snippet": "", |
| 232 | + "body": ("The following source files are >100 lines and not referenced by any knowledge doc:\n" + $files + "\n\nConsider adding knowledge documentation for the most critical of these.") |
| 233 | + }]') |
| 234 | +fi |
| 235 | + |
| 236 | +# ── Phase 4: Post Results ──────────────────────────────────────────────── |
| 237 | + |
| 238 | +TOTAL=$(echo "$ALL_FINDINGS" | jq 'length' 2>/dev/null || echo 0) |
| 239 | +echo "" |
| 240 | +echo "=== Total findings: ${TOTAL} ===" |
| 241 | + |
| 242 | +if [ "$TOTAL" -eq 0 ]; then |
| 243 | + echo "All knowledge files are accurate. Exiting." |
| 244 | + exit 0 |
| 245 | +fi |
| 246 | + |
| 247 | +# Build issue body |
| 248 | +ISSUE_BODY="## 📚 Knowledge Accuracy Audit |
| 249 | +
|
| 250 | +**Commit:** \`${COMMIT_SHA:0:8}\` |
| 251 | +**Knowledge files checked:** ${#KNOWLEDGE_FILES[@]} |
| 252 | +**Source files in repo:** ${SOURCE_COUNT} |
| 253 | +**Total findings:** ${TOTAL} |
| 254 | +
|
| 255 | +--- |
| 256 | +" |
| 257 | + |
| 258 | +# Group by severity |
| 259 | +for severity in critical important suggestion; do |
| 260 | + sev_findings=$(echo "$ALL_FINDINGS" | jq -c --arg s "$severity" '[.[] | select(.severity == $s)]') |
| 261 | + sev_count=$(echo "$sev_findings" | jq 'length') |
| 262 | + [ "$sev_count" -eq 0 ] && continue |
| 263 | + |
| 264 | + case "$severity" in |
| 265 | + critical) icon="🔴"; label="Critical — Factual Errors" ;; |
| 266 | + important) icon="🟠"; label="Important — Stale or Missing" ;; |
| 267 | + suggestion) icon="🔵"; label="Suggestions — Coverage Gaps" ;; |
| 268 | + esac |
| 269 | + |
| 270 | + ISSUE_BODY="${ISSUE_BODY} |
| 271 | +### ${icon} ${label} (${sev_count}) |
| 272 | +
|
| 273 | +" |
| 274 | + |
| 275 | + findings_text=$(echo "$sev_findings" | jq -r ' |
| 276 | + .[] | |
| 277 | + "#### " + .title + "\n" + |
| 278 | + "📁 `" + .file + "`\n" + |
| 279 | + .body + "\n" + |
| 280 | + (if .snippet != "" then "\n> " + (.snippet | gsub("\n"; "\n> ")) + "\n" else "" end) + |
| 281 | + "\n---\n" |
| 282 | + ' 2>/dev/null || true) |
| 283 | + |
| 284 | + ISSUE_BODY="${ISSUE_BODY}${findings_text}" |
| 285 | +done |
| 286 | + |
| 287 | +# Add provenance |
| 288 | +ISSUE_BODY="${ISSUE_BODY} |
| 289 | +
|
| 290 | +<details> |
| 291 | +<summary>Audit Provenance</summary> |
| 292 | +
|
| 293 | +- **Model:** ${MODEL} |
| 294 | +- **Commit:** ${COMMIT_SHA} |
| 295 | +- **Knowledge files:** ${#KNOWLEDGE_FILES[@]} |
| 296 | +- **Source files scanned:** ${SOURCE_COUNT} |
| 297 | +- **Uncovered significant files:** ${UNCOVERED_COUNT} |
| 298 | +
|
| 299 | +</details>" |
| 300 | + |
| 301 | +# Post as rolling issue |
| 302 | +AUDIT_LABEL="audit:knowledge-accuracy" |
| 303 | +gh label create "$AUDIT_LABEL" --description "Knowledge documentation accuracy audit" --color "7057ff" --repo "$REPO" 2>/dev/null || true |
| 304 | + |
| 305 | +EXISTING=$(gh issue list --repo "$REPO" --label "$AUDIT_LABEL" --state open --json number --jq '.[0].number' 2>/dev/null || true) |
| 306 | + |
| 307 | +if [ -n "$EXISTING" ] && [ "$EXISTING" != "null" ]; then |
| 308 | + echo "Updating existing issue #${EXISTING}..." |
| 309 | + gh issue comment "$EXISTING" --repo "$REPO" --body "$ISSUE_BODY" |
| 310 | + echo "Updated: https://github.com/${REPO}/issues/${EXISTING}" |
| 311 | +else |
| 312 | + echo "Creating new issue..." |
| 313 | + ISSUE_URL=$(gh issue create --repo "$REPO" \ |
| 314 | + --title "📚 Knowledge Accuracy Audit" \ |
| 315 | + --label "$AUDIT_LABEL" \ |
| 316 | + --body "$ISSUE_BODY" 2>&1) |
| 317 | + echo "Created: ${ISSUE_URL}" |
| 318 | +fi |
| 319 | + |
| 320 | +echo "=== Knowledge accuracy audit complete ===" |
0 commit comments