From 8d2cc29f427f6efacbaf559aa12b336cdb0a3099 Mon Sep 17 00:00:00 2001 From: Mufacoderz Date: Sun, 14 Jun 2026 16:01:54 +0800 Subject: [PATCH] feat: optimize context ranking and token budget Co-authored-by: devmap-agent <238585242+devmap-agent@users.noreply.github.com> --- docs/architecture.md | 22 ++- docs/for-me-personal/PROGRESS.md | 15 ++ docs/for-me-personal/TEST.md | 29 ++++ packages/cli/src/ai/contextBuilder.ts | 159 ++++++++++++++++++++-- packages/cli/test/context-builder.test.ts | 109 ++++++++++++++- 5 files changed, 320 insertions(+), 14 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 3473982..b3332a8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -438,10 +438,24 @@ app/api/auth/* | Limit | Value | | -------------------- | ------------------------- | -| Preferred file count | 3–5 files | -| Maximum file count | 5 files | -| Large file behavior | Extract relevant sections | -| Full project source | Never sent | +| Preferred file count | 3–5 files | +| Maximum file count | 5 files | +| English navigation queries | 2 files, 60 lines each | +| Large file behavior | Extract relevant sections | +| Full project source | Never sent | + +Test files and fixtures are excluded from normal product questions. They are +eligible when an English query explicitly mentions testing terms such as +`test`, `spec`, `fixture`, or `coverage`. + +Explicit English scope terms provide a ranking boost: + +* `cli`, `command`, `terminal` +* `web`, `ui`, `frontend`, `component`, `page` +* `docs`, `documentation`, `readme` + +Scope matching is a boost rather than a hard exclusion so cross-package +dependencies can still be selected when their direct relevance is stronger. --- diff --git a/docs/for-me-personal/PROGRESS.md b/docs/for-me-personal/PROGRESS.md index 6d70005..b270d5f 100644 --- a/docs/for-me-personal/PROGRESS.md +++ b/docs/for-me-personal/PROGRESS.md @@ -4,6 +4,21 @@ Terakhir diperbarui: 2026-06-14 ## Update 2026-06-14 +### Context Builder Token Optimization + +- Pertanyaan navigasi English seperti `where` dan `find` sekarang memakai + maksimal dua file dengan maksimal 60 baris per file. +- Test, spec, dan fixture dikeluarkan dari pertanyaan produk biasa sehingga + dummy authentication fixture tidak lagi dianggap sebagai fitur production. +- File test dapat dipilih kembali ketika query English menyebut `test`, + `testing`, `spec`, `fixture`, atau `coverage`. +- Scope English untuk CLI, web UI, dan documentation memberi ranking boost + tanpa melakukan hard exclusion terhadap package lain. +- Benchmark existing tetap mencapai top-1 accuracy 20/20 dan top-3 recall + 20/20. +- Payload `where scanner` pada snapshot DevMap turun dari sekitar 20.844 + karakter menjadi 4.423 karakter sebelum tokenisasi. + ### Reliability Fixes - Groq HTTP 429 sekarang di-retry maksimal tiga kali dengan exponential diff --git a/docs/for-me-personal/TEST.md b/docs/for-me-personal/TEST.md index b4b2359..61f8487 100644 --- a/docs/for-me-personal/TEST.md +++ b/docs/for-me-personal/TEST.md @@ -15,6 +15,35 @@ Ada beberapa versi DevMap yang dapat diuji: | npm link | CLI global sementara | Menguji command `devmap` dari folder mana pun | | CI/runtime | OS dan versi Node berbeda | Verifikasi lintas platform sebelum release | +## Context Builder Ranking + +Jalankan focused test ranking dan evaluation: + +```powershell +pnpm --filter devmap exec tsx --test test/context-builder.test.ts test/context-builder-eval.test.ts +``` + +Expected result: + +- Pertanyaan produk tidak memilih `test/`, `tests/`, `__tests__/`, fixture, + `*.test.*`, atau `*.spec.*`. +- Pertanyaan testing dalam English dapat memilih file tersebut. +- Pertanyaan navigasi English memilih maksimal dua file dan 60 baris per file. +- Istilah CLI dan web UI memprioritaskan package yang sesuai. +- Evaluation tetap top-1 accuracy 20/20 dan top-3 recall 20/20. + +Manual source-mode check: + +```powershell +pnpm dev:cli ask "where scanner" +pnpm dev:cli ask "which tests cover the scanner?" +pnpm dev:cli ask "where is the web UI dashboard component?" +``` + +Periksa `Relevant Files` dan prompt token usage. Query pertama seharusnya +memprioritaskan production CLI source dan memakai context jauh lebih kecil +daripada default lama lima file dengan maksimal 200 baris per file. + ## Urutan Testing Yang Direkomendasikan Untuk development harian: diff --git a/packages/cli/src/ai/contextBuilder.ts b/packages/cli/src/ai/contextBuilder.ts index a610972..19a6963 100644 --- a/packages/cli/src/ai/contextBuilder.ts +++ b/packages/cli/src/ai/contextBuilder.ts @@ -4,6 +4,8 @@ import type { ProjectMap } from "../analyzers/projectMap.js"; const DEFAULT_MAX_FILES = 5; const DEFAULT_MAX_LINES_PER_FILE = 200; +const NAVIGATION_MAX_FILES = 2; +const NAVIGATION_MAX_LINES_PER_FILE = 60; const STOP_WORDS = new Set([ "about", @@ -13,6 +15,8 @@ const STOP_WORDS = new Set([ "bekerja", "dalam", "does", + "find", + "have", "dimana", "dengan", "from", @@ -27,6 +31,23 @@ const STOP_WORDS = new Set([ "yang" ]); +const TEST_QUERY_TERMS = new Set([ + "coverage", + "fixture", + "fixtures", + "spec", + "specs", + "test", + "testing", + "tests" +]); + +const SCOPE_QUERY_TERMS = { + cli: new Set(["cli", "command", "commands", "terminal"]), + docs: new Set(["documentation", "docs", "readme"]), + web: new Set(["component", "frontend", "page", "ui", "web"]) +} as const; + const CONCEPT_ALIASES: Record = { auth: [ "auth", @@ -75,6 +96,12 @@ type RankedFile = { reasons: string[]; }; +type QueryProfile = { + includeTests: boolean; + isNavigation: boolean; + scopes: Set; +}; + export async function buildQuestionContext( projectRoot: string, snapshot: ProjectMap, @@ -82,12 +109,19 @@ export async function buildQuestionContext( options: ContextBuilderOptions = {} ): Promise { const keywords = extractContextKeywords(question); - const maxFiles = normalizeLimit(options.maxFiles, DEFAULT_MAX_FILES); + const profile = classifyQuery(question); + const defaultMaxFiles = profile.isNavigation + ? NAVIGATION_MAX_FILES + : DEFAULT_MAX_FILES; + const defaultMaxLines = profile.isNavigation + ? NAVIGATION_MAX_LINES_PER_FILE + : DEFAULT_MAX_LINES_PER_FILE; + const maxFiles = normalizeLimit(options.maxFiles, defaultMaxFiles); const maxLinesPerFile = normalizeLimit( options.maxLinesPerFile, - DEFAULT_MAX_LINES_PER_FILE + defaultMaxLines ); - const rankedFiles = rankContextFiles(snapshot, keywords); + const rankedFiles = rankContextFiles(snapshot, keywords, profile); const files: ContextFile[] = []; for (const rankedFile of rankedFiles) { @@ -130,10 +164,18 @@ export function extractContextKeywords(question: string): string[] { return [...keywords]; } -function rankContextFiles(snapshot: ProjectMap, keywords: string[]): RankedFile[] { +function rankContextFiles( + snapshot: ProjectMap, + keywords: string[], + profile: QueryProfile +): RankedFile[] { const ranked = new Map(); for (const [path, metadata] of Object.entries(snapshot.fileIndex)) { + if (!profile.includeTests && isTestPath(path)) { + continue; + } + const reasons: string[] = []; const normalizedPath = path.toLowerCase(); const symbols = metadata.exportedSymbols.join(" ").toLowerCase(); @@ -165,6 +207,10 @@ function rankContextFiles(snapshot: ProjectMap, keywords: string[]): RankedFile[ score += routeScore.score; reasons.push(...routeScore.reasons); + const scopeScore = scoreScopeEvidence(path, profile.scopes); + score += scopeScore.score; + reasons.push(...scopeScore.reasons); + const criticalFile = snapshot.criticalFiles.find((file) => file.path === path); if (criticalFile && score > 0) { score += Math.min(criticalFile.score, 5); @@ -180,10 +226,10 @@ function rankContextFiles(snapshot: ProjectMap, keywords: string[]): RankedFile[ } } - expandGraphNeighbors(snapshot, ranked); + expandGraphNeighbors(snapshot, ranked, profile); if (ranked.size === 0) { - addFallbackFiles(snapshot, ranked); + addFallbackFiles(snapshot, ranked, profile); } return [...ranked.values()] @@ -233,7 +279,8 @@ function scoreRouteEvidence( function expandGraphNeighbors( snapshot: ProjectMap, - ranked: Map + ranked: Map, + profile: QueryProfile ): void { const directMatches = [...ranked.values()] .sort((left, right) => right.score - left.score) @@ -242,6 +289,10 @@ function expandGraphNeighbors( for (const match of directMatches) { const imports = snapshot.fileIndex[match.path]?.imports ?? []; for (const importedPath of imports) { + if (!profile.includeTests && isTestPath(importedPath)) { + continue; + } + addRelatedFile( ranked, importedPath, @@ -251,6 +302,10 @@ function expandGraphNeighbors( } for (const [candidatePath, metadata] of Object.entries(snapshot.fileIndex)) { + if (!profile.includeTests && isTestPath(candidatePath)) { + continue; + } + if (metadata.imports.includes(match.path)) { addRelatedFile( ranked, @@ -263,6 +318,87 @@ function expandGraphNeighbors( } } +function classifyQuery(question: string): QueryProfile { + const terms = new Set( + question + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean) + ); + const scopes = new Set(); + + for (const [scope, scopeTerms] of Object.entries(SCOPE_QUERY_TERMS) as Array< + [keyof typeof SCOPE_QUERY_TERMS, Set] + >) { + if ([...scopeTerms].some((term) => terms.has(term))) { + scopes.add(scope); + } + } + + return { + includeTests: [...TEST_QUERY_TERMS].some((term) => terms.has(term)), + isNavigation: terms.has("where") || terms.has("find"), + scopes + }; +} + +function scoreScopeEvidence( + path: string, + scopes: Set +): Pick { + const normalizedPath = path.toLowerCase().replaceAll("\\", "/"); + let score = 0; + const reasons: string[] = []; + + if ( + scopes.has("cli") + && ( + normalizedPath.includes("/cli/") + || normalizedPath.startsWith("cli/") + || normalizedPath.includes("/commands/") + ) + ) { + score += 8; + reasons.push("matches CLI scope"); + } + + if ( + scopes.has("web") + && ( + normalizedPath.includes("/web/") + || normalizedPath.includes("/components/") + || normalizedPath.includes("/pages/") + || normalizedPath.includes("/app/") + ) + ) { + score += 8; + reasons.push("matches web scope"); + } + + if ( + scopes.has("docs") + && (normalizedPath.startsWith("docs/") || normalizedPath.endsWith(".md")) + ) { + score += 8; + reasons.push("matches documentation scope"); + } + + return { score, reasons }; +} + +function isTestPath(path: string): boolean { + const normalizedPath = path.toLowerCase().replaceAll("\\", "/"); + return ( + normalizedPath.startsWith("test/") + || normalizedPath.startsWith("tests/") + || normalizedPath.includes("/test/") + || normalizedPath.includes("/tests/") + || normalizedPath.includes("/__tests__/") + || normalizedPath.includes("/fixtures/") + || /\.(?:test|spec)\.[cm]?[jt]sx?$/.test(normalizedPath) + ); +} + function addRelatedFile( ranked: Map, path: string, @@ -288,9 +424,14 @@ function addRelatedFile( function addFallbackFiles( snapshot: ProjectMap, - ranked: Map + ranked: Map, + profile: QueryProfile ): void { - for (const criticalFile of snapshot.criticalFiles.slice(0, DEFAULT_MAX_FILES)) { + const fallbackFiles = snapshot.criticalFiles + .filter((file) => profile.includeTests || !isTestPath(file.path)) + .slice(0, DEFAULT_MAX_FILES); + + for (const criticalFile of fallbackFiles) { ranked.set(criticalFile.path, { path: criticalFile.path, score: criticalFile.score, diff --git a/packages/cli/test/context-builder.test.ts b/packages/cli/test/context-builder.test.ts index 802f969..f32cdaf 100644 --- a/packages/cli/test/context-builder.test.ts +++ b/packages/cli/test/context-builder.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -122,3 +122,110 @@ test("context builder enforces file limits and rejects paths outside the project await rm(outsidePath, { force: true }); } }); + +test("context builder excludes test fixtures from product questions", async () => { + const projectRoot = await createScopedContextProject(); + + try { + const snapshot = await createProjectMap(projectRoot); + const context = await buildQuestionContext( + projectRoot, + snapshot, + "Does this project have authentication?" + ); + + assert.equal(context.files[0]?.path, "src/auth.ts"); + assert.ok(context.files.every((file) => !file.path.includes("test/fixtures"))); + assert.ok(context.files.every((file) => !file.path.endsWith(".test.ts"))); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } +}); + +test("context builder excludes tests from fallback context", async () => { + const projectRoot = await createScopedContextProject(); + + try { + const snapshot = await createProjectMap(projectRoot); + snapshot.criticalFiles = [ + { path: "src/auth.test.ts", score: 20, reasons: ["fixture critical file"] }, + { path: "src/auth.ts", score: 10, reasons: ["production critical file"] } + ]; + const context = await buildQuestionContext( + projectRoot, + snapshot, + "Explain the zqxv subsystem" + ); + + assert.ok(context.files.some((file) => file.path === "src/auth.ts")); + assert.ok(context.files.every((file) => !file.path.endsWith(".test.ts"))); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } +}); + +test("context builder includes tests only when the English query requests them", async () => { + const projectRoot = await createScopedContextProject(); + + try { + const snapshot = await createProjectMap(projectRoot); + const context = await buildQuestionContext( + projectRoot, + snapshot, + "Which tests and fixtures cover authentication?" + ); + + assert.ok(context.files.some((file) => file.path.endsWith("auth.test.ts"))); + assert.ok(context.files.some((file) => file.path.includes("test/fixtures"))); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } +}); + +test("context builder boosts explicit CLI and web scopes without hard exclusion", async () => { + const projectRoot = await createScopedContextProject(); + + try { + const snapshot = await createProjectMap(projectRoot); + const cliContext = await buildQuestionContext( + projectRoot, + snapshot, + "Where is the CLI dashboard command?" + ); + const webContext = await buildQuestionContext( + projectRoot, + snapshot, + "Where is the web UI dashboard component?" + ); + + assert.equal(cliContext.files[0]?.path, "packages/cli/src/dashboard.ts"); + assert.equal(webContext.files[0]?.path, "apps/web/src/Dashboard.ts"); + assert.ok(cliContext.files.length <= 2); + assert.ok(webContext.files.length <= 2); + assert.ok(cliContext.files.every((file) => + file.content.split(/\r?\n/).length <= 60 + )); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } +}); + +async function createScopedContextProject(): Promise { + const projectRoot = await mkdtemp(join(tmpdir(), "devmap-context-scope-")); + const files = { + "package.json": JSON.stringify({ name: "context-scope" }), + "src/auth.ts": "export function authenticateUser() { return true; }\n", + "src/auth.test.ts": "export function authenticationTest() { return true; }\n", + "test/fixtures/auth.ts": "export function fixtureAuthentication() { return true; }\n", + "packages/cli/src/dashboard.ts": "export function dashboardCommand() { return true; }\n", + "apps/web/src/Dashboard.ts": "export function DashboardComponent() { return true; }\n" + }; + + for (const [path, content] of Object.entries(files)) { + const target = join(projectRoot, path); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, content, "utf8"); + } + + return projectRoot; +}