From 3edb85de0e8f5a13750edf6a29915d73272e9648 Mon Sep 17 00:00:00 2001 From: manojmallick Date: Sun, 5 Apr 2026 00:50:28 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20v2.3.0=20=E2=80=94=20query-aware=20retr?= =?UTF-8?q?ieval=20+=20query=5Fcontext=20MCP=20tool=20(closes=20#8)=20(#10?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 24 +++ gen-context.js | 196 ++++++++++++++++- package.json | 2 +- src/config/defaults.js | 8 + src/mcp/handlers.js | 29 ++- src/mcp/server.js | 5 +- src/mcp/tools.js | 24 +++ src/retrieval/ranker.js | 242 +++++++++++++++++++++ src/retrieval/tokenizer.js | 54 +++++ test/integration/analyze.test.js | 8 +- test/integration/mcp-server.test.js | 5 +- test/integration/mcp-v14.test.js | 5 +- test/integration/retrieval.test.js | 315 ++++++++++++++++++++++++++++ 13 files changed, 902 insertions(+), 15 deletions(-) create mode 100644 src/retrieval/ranker.js create mode 100644 src/retrieval/tokenizer.js create mode 100644 test/integration/retrieval.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index bd0d73c8..6621bccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,30 @@ Format: [Semantic Versioning](https://semver.org/) --- +## [2.3.0] — 2026-04-07 + +### Added +- **Query-aware retrieval** — `src/retrieval/tokenizer.js` and `src/retrieval/ranker.js`: zero-dependency relevance ranker that scores every file against a free-text query by exact token, symbol, prefix, path, and recency signals. +- **`--query ""` CLI flag** — ranks all context files by relevance and prints a scored table (Rank | File | Score | Sigs | Tokens) plus the top-3 signature blocks; `--query "" --json` for machine-readable output; `--query "" --top ` to limit result set. +- **`query_context` MCP tool** — 8th MCP tool; accepts `{ query: string, topK?: number }` and returns the same ranked table as the `--query` CLI flag; live within any running MCP session. +- **Retrieval config** — `config.retrieval.topK` (default 10) and `config.retrieval.recencyBoost` (default 1.5×) added to `src/config/defaults.js`. +- **`test/integration/retrieval.test.js`** — 23 integration tests covering tokenizer unit tests, ranker sorting/scoring/topK/empty-query, `formatRankTable`, `formatRankJSON`, CLI `--query` flags, and MCP `query_context`. + +### Changed +- `src/mcp/server.js` version bumped to `2.3.0`. +- `test/integration/mcp-server.test.js` and `mcp-v14.test.js` updated to assert 8 tools. +- `test/integration/analyze.test.js` version assertion updated to `2.3.0`. + +### Validation gate +- 21/21 extractor unit tests passed +- 20/20 integration suites passed (0 failures) +- `node gen-context.js --version` → `2.3.0` +- `node gen-context.js --query "python extractor"` → `src/extractors/python.js` in top-3 +- `node gen-context.js --query "fix secret scanning" --json` → valid JSON +- MCP `tools/list` → 8 tools including `query_context` + +--- + ## [2.2.0] — 2026-04-06 ### Added diff --git a/gen-context.js b/gen-context.js index 7567cbd4..270eae70 100755 --- a/gen-context.js +++ b/gen-context.js @@ -2879,7 +2879,23 @@ __factories["./src/mcp/handlers"] = function(module, exports) { ].join('\n'); } - module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules }; + function queryContext(args, cwd) { + if (!args || !args.query) return 'Missing required argument: query'; + const contextPath = path.join(cwd, CONTEXT_FILE); + if (!fs.existsSync(contextPath)) return 'No context file found. Run: node gen-context.js'; + try { + const { rank, buildSigIndex, formatRankTable } = __require('./src/retrieval/ranker'); + const index = buildSigIndex(cwd); + if (index.size === 0) return 'No signatures indexed. Run: node gen-context.js'; + const topK = Math.min(Math.max(1, parseInt(args.topK, 10) || 10), 25); + const results = rank(args.query, index, { topK }); + return formatRankTable(results, args.query); + } catch (err) { + return `_query_context failed: ${err.message}_`; + } + } + + module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext }; }; // ── ./src/mcp/server ── @@ -2899,7 +2915,7 @@ __factories["./src/mcp/server"] = function(module, exports) { const readline = require('readline'); const { TOOLS } = __require('./src/mcp/tools'); - const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules } = __require('./src/mcp/handlers'); + const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext } = __require('./src/mcp/handlers'); const SERVER_INFO = { name: 'sigmap', @@ -2958,6 +2974,7 @@ __factories["./src/mcp/server"] = function(module, exports) { else if (name === 'get_routing') text = getRouting(args, cwd); else if (name === 'explain_file') text = explainFile(args, cwd); else if (name === 'list_modules') text = listModules(args, cwd); + else if (name === 'query_context') text = queryContext(args, cwd); else { respondError(id, -32601, `Unknown tool: ${name}`); return; @@ -3137,6 +3154,30 @@ __factories["./src/mcp/tools"] = function(module, exports) { required: [], }, }, + { + name: 'query_context', + description: + 'Rank and return the most relevant files for a specific task or question. ' + + 'Uses keyword + symbol + path scoring to surface only the top-K files relevant ' + + 'to the query — much cheaper than reading all context. ' + + 'Returns ranked file list with signatures and relevance scores.', + inputSchema: { + type: 'object', + properties: { + query: { + type: 'string', + description: + 'Natural language task description or keyword(s) to rank files against. ' + + 'E.g. "add a new language extractor", "fix secret scanning", "auth module".', + }, + topK: { + type: 'number', + description: 'Maximum number of files to return (default: 10, max: 25).', + }, + }, + required: ['query'], + }, + }, ]; module.exports = { TOOLS }; @@ -3570,6 +3611,120 @@ __factories["./src/tracking/logger"] = function(module, exports) { }; +// ── ./src/retrieval/tokenizer ── +__factories["./src/retrieval/tokenizer"] = function(module, exports) { + 'use strict'; + const STOP_WORDS = new Set([ + 'the', 'a', 'an', 'in', 'of', 'to', 'for', 'and', 'or', 'is', 'are', + 'that', 'this', 'it', 'with', 'from', 'by', 'be', 'as', 'on', 'at', + 'do', 'not', 'use', 'get', 'set', 'up', 'if', 'no', 'so', 'we', + ]); + function tokenize(text, opts) { + if (!text || typeof text !== 'string') return []; + const removeStop = opts && opts.removeStopWords === false ? false : true; + const minLen = (opts && opts.minLength) || 2; + const tokens = text + .replace(/\.\w{1,6}(?=\s|\/|$)/g, ' ') + .replace(/([a-z])([A-Z])/g, '$1 $2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + .replace(/[_\-\.\/]/g, ' ') + .replace(/[^\w\s]/g, ' ') + .toLowerCase() + .split(/\s+/) + .filter((t) => t.length >= minLen); + if (!removeStop) return [...new Set(tokens)]; + return [...new Set(tokens.filter((t) => !STOP_WORDS.has(t)))]; + } + module.exports = { tokenize, STOP_WORDS }; +}; + +// ── ./src/retrieval/ranker ── +__factories["./src/retrieval/ranker"] = function(module, exports) { + 'use strict'; + const { tokenize, STOP_WORDS } = __require('./src/retrieval/tokenizer'); + const DEFAULT_WEIGHTS = { + exactToken: 1.0, symbolMatch: 0.5, prefixMatch: 0.3, pathMatch: 0.8, recencyBoost: 1.5, + }; + function scoreFile(filePath, sigs, queryTokens, weights) { + if (!sigs || sigs.length === 0) return 0; + const w = weights || DEFAULT_WEIGHTS; + const sigTokenSet = new Set(tokenize(sigs.join(' '))); + const pathTokenSet = new Set(tokenize(filePath)); + let score = 0; + for (const qt of queryTokens) { + if (STOP_WORDS.has(qt)) continue; + if (sigTokenSet.has(qt)) { + score += w.exactToken; + if (sigs.some((sig) => tokenize(sig.replace(/[^a-zA-Z0-9_\s]/g, ' ')).includes(qt))) score += w.symbolMatch; + } + if (qt.length >= 4) { + for (const st of sigTokenSet) { + if (st !== qt && st.startsWith(qt)) { score += w.prefixMatch; break; } + } + } + if (pathTokenSet.has(qt)) score += w.pathMatch; + } + return score; + } + function rank(query, sigIndex, opts) { + if (!query || typeof query !== 'string') return []; + if (!sigIndex || !(sigIndex instanceof Map) || sigIndex.size === 0) return []; + const topK = (opts && opts.topK) || 10; + const recencyMultiplier = (opts && opts.recencyBoost) || DEFAULT_WEIGHTS.recencyBoost; + const recencySet = (opts && opts.recencySet) || null; + const weights = (opts && opts.weights) ? Object.assign({}, DEFAULT_WEIGHTS, opts.weights) : DEFAULT_WEIGHTS; + const queryTokens = tokenize(query); + if (queryTokens.length === 0) { + const all = []; + for (const [file, sigs] of sigIndex.entries()) all.push({ file, score: sigs.length, sigs, tokens: Math.ceil(sigs.join('\n').length / 4) }); + all.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file)); + return all.slice(0, topK); + } + const scored = []; + for (const [file, sigs] of sigIndex.entries()) { + let score = scoreFile(file, sigs, queryTokens, weights); + if (recencySet && recencySet.has(file) && score > 0) score *= recencyMultiplier; + scored.push({ file, score, sigs, tokens: Math.ceil(sigs.join('\n').length / 4) }); + } + scored.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file)); + return scored.slice(0, topK); + } + function buildSigIndex(cwd) { + const fs = require('fs'); const path = require('path'); + const contextPath = path.join(cwd, '.github', 'copilot-instructions.md'); + const index = new Map(); + if (!fs.existsSync(contextPath)) return index; + const content = fs.readFileSync(contextPath, 'utf8'); + const lines = content.split('\n'); + let currentFile = null; let inBlock = false; let sigs = []; + for (const line of lines) { + const hm = line.match(/^###\s+(\S+)\s*$/); + if (hm) { if (currentFile !== null) index.set(currentFile, sigs); currentFile = hm[1]; sigs = []; inBlock = false; continue; } + if (line.startsWith('```')) { inBlock = !inBlock; continue; } + if (inBlock && currentFile && line.trim()) sigs.push(line.trim()); + } + if (currentFile !== null) index.set(currentFile, sigs); + return index; + } + function formatRankTable(results, query) { + if (!results || results.length === 0) return `No matching files found for query: "${query}"\n`; + const lines = [`## Query: ${query}`, '', '| Rank | File | Score | Sigs | Tokens |', '|------|------|-------|------|--------|', + ...results.map((r, i) => `| ${i + 1} | ${r.file} | ${r.score.toFixed(2)} | ${r.sigs.length} | ${r.tokens} |`), '']; + for (const r of results.slice(0, 3)) { + if (r.sigs.length > 0) { + lines.push(`### ${r.file}`, '```', ...r.sigs.slice(0, 10)); + if (r.sigs.length > 10) lines.push(`... (${r.sigs.length - 10} more)`); + lines.push('```', ''); + } + } + return lines.join('\n'); + } + function formatRankJSON(results, query) { + return { query, results: (results || []).map((r, i) => ({ rank: i + 1, file: r.file, score: r.score, sigs: r.sigs, tokens: r.tokens })), totalResults: (results || []).length }; + } + module.exports = { rank, buildSigIndex, scoreFile, formatRankTable, formatRankJSON, DEFAULT_WEIGHTS }; +}; + // ── ./src/eval/scorer ── __factories["./src/eval/scorer"] = function(module, exports) { 'use strict'; @@ -3936,7 +4091,7 @@ const path = require('path'); const os = require('os'); const { execSync } = require('child_process'); -const VERSION = '2.2.0'; +const VERSION = '2.3.0'; const MARKER = '\n\n## Auto-generated signatures\n\n'; function requireSourceOrBundled(key) { @@ -5149,6 +5304,9 @@ Usage: node gen-context.js --analyze --json Breakdown as JSON node gen-context.js --analyze --slow Re-time each extractor; flag files >50ms node gen-context.js --diagnose-extractors Run all 21 extractors vs fixtures; show pass/fail + diff + node gen-context.js --query "" Rank files by relevance to a query + node gen-context.js --query "" --json Ranked results as JSON + node gen-context.js --query "" --top Limit results to top N files (default 10) node gen-context.js --init Write example config + .contextignore scaffold node gen-context.js --help Show this message node gen-context.js --version Show version @@ -5435,6 +5593,38 @@ function main() { } } + if (args.includes('--query')) { + try { + const qIdx = args.indexOf('--query'); + const query = (args[qIdx + 1] || '').trim(); + if (!query || query.startsWith('--')) { + console.error('[sigmap] --query requires a search string'); + console.error(' Example: node gen-context.js --query "add a new language extractor"'); + process.exit(1); + } + const { rank, buildSigIndex, formatRankTable, formatRankJSON } = requireSourceOrBundled('./src/retrieval/ranker'); + const index = buildSigIndex(cwd); + if (index.size === 0) { + console.error('[sigmap] no context file found. Run: node gen-context.js'); + process.exit(1); + } + const topIdx = args.indexOf('--top'); + const topK = topIdx >= 0 ? Math.min(Math.max(1, parseInt(args[topIdx + 1], 10) || 10), 25) + : ((config && config.retrieval && config.retrieval.topK) || 10); + const recencyBoost = (config && config.retrieval && config.retrieval.recencyBoost) || 1.5; + const results = rank(query, index, { topK, recencyBoost }); + if (args.includes('--json')) { + process.stdout.write(JSON.stringify(formatRankJSON(results, query)) + '\n'); + } else { + process.stdout.write(formatRankTable(results, query)); + } + } catch (err) { + console.error(`[sigmap] query error: ${err.message}`); + process.exit(1); + } + process.exit(0); + } + if (args.includes('--report')) { if (args.includes('--history')) { try { diff --git a/package.json b/package.json index 875b6e16..fab0b6c3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sigmap", - "version": "2.2.0", + "version": "2.3.0", "description": "Zero-dependency AI context engine — 97% token reduction. No npm install. Runs on Node 18+.", "main": "gen-context.js", "bin": { diff --git a/src/config/defaults.js b/src/config/defaults.js index 3e632d2d..72f4c4fb 100644 --- a/src/config/defaults.js +++ b/src/config/defaults.js @@ -92,6 +92,14 @@ const DEFAULTS = { // Add reverse dependency usage hints on file headings (opt-in) impactRadius: false, + + // Query-aware retrieval settings (v2.3) + retrieval: { + // Maximum number of files to return for --query + topK: 10, + // Multiplier applied to recently-changed files (>1 boosts them up) + recencyBoost: 1.5, + }, }; module.exports = { DEFAULTS }; diff --git a/src/mcp/handlers.js b/src/mcp/handlers.js index a21b4c0b..9f5f00f7 100644 --- a/src/mcp/handlers.js +++ b/src/mcp/handlers.js @@ -430,4 +430,31 @@ function listModules(args, cwd) { ].join('\n'); } -module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules }; \ No newline at end of file +/** + * query_context({ query, topK? }) → string + * + * Ranks context-file entries by relevance to the query and returns the + * top-K most relevant files with their signatures and scores. + */ +function queryContext(args, cwd) { + if (!args || !args.query) return 'Missing required argument: query'; + + const contextPath = path.join(cwd, CONTEXT_FILE); + if (!fs.existsSync(contextPath)) { + return 'No context file found. Run: node gen-context.js'; + } + + try { + const { rank, buildSigIndex, formatRankTable } = require('../retrieval/ranker'); + const index = buildSigIndex(cwd); + if (index.size === 0) return 'No signatures indexed. Run: node gen-context.js'; + + const topK = Math.min(Math.max(1, parseInt(args.topK, 10) || 10), 25); + const results = rank(args.query, index, { topK }); + return formatRankTable(results, args.query); + } catch (err) { + return `_query_context failed: ${err.message}_`; + } +} + +module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext }; \ No newline at end of file diff --git a/src/mcp/server.js b/src/mcp/server.js index 3e0bdbd0..c834604c 100644 --- a/src/mcp/server.js +++ b/src/mcp/server.js @@ -14,11 +14,11 @@ const readline = require('readline'); const { TOOLS } = require('./tools'); -const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules } = require('./handlers'); +const { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext } = require('./handlers'); const SERVER_INFO = { name: 'sigmap', - version: '2.2.0', + version: '2.3.0', description: 'SigMap MCP server — code signatures on demand', }; @@ -73,6 +73,7 @@ function dispatch(msg, cwd) { else if (name === 'get_routing') text = getRouting(args, cwd); else if (name === 'explain_file') text = explainFile(args, cwd); else if (name === 'list_modules') text = listModules(args, cwd); + else if (name === 'query_context') text = queryContext(args, cwd); else { respondError(id, -32601, `Unknown tool: ${name}`); return; diff --git a/src/mcp/tools.js b/src/mcp/tools.js index 0dbee90d..fe2e3e26 100644 --- a/src/mcp/tools.js +++ b/src/mcp/tools.js @@ -120,6 +120,30 @@ const TOOLS = [ required: [], }, }, + { + name: 'query_context', + description: + 'Rank and return the most relevant files for a specific task or question. ' + + 'Uses keyword + symbol + path scoring to surface only the top-K files relevant ' + + 'to the query — much cheaper than reading all context. ' + + 'Returns ranked file list with signatures and relevance scores.', + inputSchema: { + type: 'object', + properties: { + query: { + type: 'string', + description: + 'Natural language task description or keyword(s) to rank files against. ' + + 'E.g. "add a new language extractor", "fix secret scanning", "auth module".', + }, + topK: { + type: 'number', + description: 'Maximum number of files to return (default: 10, max: 25).', + }, + }, + required: ['query'], + }, + }, ]; module.exports = { TOOLS }; diff --git a/src/retrieval/ranker.js b/src/retrieval/ranker.js new file mode 100644 index 00000000..337e6631 --- /dev/null +++ b/src/retrieval/ranker.js @@ -0,0 +1,242 @@ +'use strict'; + +/** + * SigMap zero-dependency relevance ranker. + * + * Ranks all files in a signature index against a natural-language query. + * Scoring weights: + * - keyword overlap (exact token match against sigs) + * - symbol match (token appears in a top-level identifier / function name) + * - partial prefix match (token is prefix of a sig token, length ≥ 4) + * - path relevance (query token appears in the file path) + * - recency boost (applied externally via recency map) + * + * Usage: + * const { rank } = require('./src/retrieval/ranker'); + * const results = rank(query, sigIndex, { topK: 10 }); + * // results: [{ file, score, sigs, tokens }] + */ + +const { tokenize, STOP_WORDS } = require('./tokenizer'); + +// --------------------------------------------------------------------------- +// Default weights +// --------------------------------------------------------------------------- +const DEFAULT_WEIGHTS = { + exactToken: 1.0, // query token exactly in sig tokens + symbolMatch: 0.5, // bonus if token appears in a function/class name line + prefixMatch: 0.3, // partial prefix hit (query token ≥ 4 chars) + pathMatch: 0.8, // query token appears in the file path + recencyBoost: 1.5, // multiplier applied when file is in recencySet +}; + +/** + * Score a single file against a query. + * + * @param {string} filePath - relative file path (e.g. 'src/extractors/python.js') + * @param {string[]} sigs - signature strings for this file + * @param {string[]} queryTokens - pre-tokenized query + * @param {object} weights + * @returns {number} + */ +function scoreFile(filePath, sigs, queryTokens, weights) { + if (!sigs || sigs.length === 0) return 0; + + const w = weights || DEFAULT_WEIGHTS; + + // Build token set from all signatures + const sigText = sigs.join(' '); + const sigTokenSet = new Set(tokenize(sigText)); + + // Build token set from the file path + const pathTokenSet = new Set(tokenize(filePath)); + + let score = 0; + + for (const qt of queryTokens) { + if (STOP_WORDS.has(qt)) continue; + + // Exact token match in sigs + if (sigTokenSet.has(qt)) { + score += w.exactToken; + + // Bonus: appears directly in a function/class/method name line + const nameLineMatch = sigs.some((sig) => { + const nt = tokenize(sig.replace(/[^a-zA-Z0-9_\s]/g, ' ')); + return nt.includes(qt); + }); + if (nameLineMatch) score += w.symbolMatch; + } + + // Prefix match (e.g. query "python" matches "pythonDeps") + if (qt.length >= 4) { + for (const st of sigTokenSet) { + if (st !== qt && st.startsWith(qt)) { + score += w.prefixMatch; + break; // one bonus per query token + } + } + } + + // Path token match + if (pathTokenSet.has(qt)) { + score += w.pathMatch; + } + } + + return score; +} + +/** + * Rank all files in a signature index against a query. + * + * @param {string} query - natural language query + * @param {Map} sigIndex - Map + * @param {object} [opts] + * @param {number} [opts.topK=10] - max results to return + * @param {number} [opts.recencyBoost=1.5] - multiplier for recent files + * @param {Set} [opts.recencySet] - set of recently-changed file paths + * @param {object} [opts.weights] - override scoring weights + * @returns {{ file: string, score: number, sigs: string[], tokens: number }[]} + */ +function rank(query, sigIndex, opts) { + if (!query || typeof query !== 'string') return []; + if (!sigIndex || !(sigIndex instanceof Map) || sigIndex.size === 0) return []; + + const topK = (opts && opts.topK) || 10; + const recencyMultiplier = (opts && opts.recencyBoost) || DEFAULT_WEIGHTS.recencyBoost; + const recencySet = (opts && opts.recencySet) || null; + const weights = (opts && opts.weights) ? Object.assign({}, DEFAULT_WEIGHTS, opts.weights) : DEFAULT_WEIGHTS; + + const queryTokens = tokenize(query); + if (queryTokens.length === 0) { + // Empty query: return top-K by file count (most signatures = most useful) + const all = []; + for (const [file, sigs] of sigIndex.entries()) { + all.push({ file, score: sigs.length, sigs, tokens: Math.ceil(sigs.join('\n').length / 4) }); + } + all.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file)); + return all.slice(0, topK); + } + + const scored = []; + for (const [file, sigs] of sigIndex.entries()) { + let score = scoreFile(file, sigs, queryTokens, weights); + + // Recency boost + if (recencySet && recencySet.has(file) && score > 0) { + score *= recencyMultiplier; + } + + scored.push({ + file, + score, + sigs, + tokens: Math.ceil(sigs.join('\n').length / 4), + }); + } + + scored.sort((a, b) => b.score - a.score || a.file.localeCompare(b.file)); + return scored.slice(0, topK); +} + +/** + * Build a signature index from the generated context file. + * Returns Map where filePath is the relative path + * as it appears in the ### headers of copilot-instructions.md. + * + * @param {string} cwd + * @returns {Map} + */ +function buildSigIndex(cwd) { + const fs = require('fs'); + const path = require('path'); + const contextPath = path.join(cwd, '.github', 'copilot-instructions.md'); + const index = new Map(); + + if (!fs.existsSync(contextPath)) return index; + + const content = fs.readFileSync(contextPath, 'utf8'); + const lines = content.split('\n'); + + let currentFile = null; + let inBlock = false; + let sigs = []; + + for (const line of lines) { + const headerMatch = line.match(/^###\s+(\S+)\s*$/); + if (headerMatch) { + if (currentFile !== null) index.set(currentFile, sigs); + currentFile = headerMatch[1]; + sigs = []; + inBlock = false; + continue; + } + if (line.startsWith('```')) { inBlock = !inBlock; continue; } + if (inBlock && currentFile && line.trim()) sigs.push(line.trim()); + } + if (currentFile !== null) index.set(currentFile, sigs); + + return index; +} + +/** + * Format ranked results as a markdown table string. + * + * @param {{ file: string, score: number, sigs: string[], tokens: number }[]} results + * @param {string} query + * @returns {string} + */ +function formatRankTable(results, query) { + if (!results || results.length === 0) { + return `No matching files found for query: "${query}"\n`; + } + + const lines = [ + `## Query: ${query}`, + '', + '| Rank | File | Score | Sigs | Tokens |', + '|------|------|-------|------|--------|', + ...results.map((r, i) => + `| ${i + 1} | ${r.file} | ${r.score.toFixed(2)} | ${r.sigs.length} | ${r.tokens} |` + ), + '', + ]; + + // Add signature details for top results + for (const r of results.slice(0, 3)) { + if (r.sigs.length > 0) { + lines.push(`### ${r.file}`); + lines.push('```'); + lines.push(...r.sigs.slice(0, 10)); + if (r.sigs.length > 10) lines.push(`... (${r.sigs.length - 10} more)`); + lines.push('```'); + lines.push(''); + } + } + + return lines.join('\n'); +} + +/** + * Format ranked results as a structured JSON-serialisable object. + * + * @param {{ file: string, score: number, sigs: string[], tokens: number }[]} results + * @param {string} query + * @returns {object} + */ +function formatRankJSON(results, query) { + return { + query, + results: (results || []).map((r, i) => ({ + rank: i + 1, + file: r.file, + score: r.score, + sigs: r.sigs, + tokens: r.tokens, + })), + totalResults: (results || []).length, + }; +} + +module.exports = { rank, buildSigIndex, scoreFile, formatRankTable, formatRankJSON, DEFAULT_WEIGHTS }; diff --git a/src/retrieval/tokenizer.js b/src/retrieval/tokenizer.js new file mode 100644 index 00000000..16ae33ef --- /dev/null +++ b/src/retrieval/tokenizer.js @@ -0,0 +1,54 @@ +'use strict'; + +/** + * SigMap zero-dependency tokenizer. + * Splits code identifiers: camelCase, snake_case, kebab-case, PascalCase, + * removes stop words, and returns lower-case tokens. + */ + +const STOP_WORDS = new Set([ + 'the', 'a', 'an', 'in', 'of', 'to', 'for', 'and', 'or', 'is', 'are', + 'that', 'this', 'it', 'with', 'from', 'by', 'be', 'as', 'on', 'at', + 'do', 'not', 'use', 'get', 'set', 'up', 'if', 'no', 'so', 'we', +]); + +/** + * Tokenize any text (query or code signature) into unique lower-case tokens. + * Handles: + * - camelCase → ['camel', 'case'] + * - PascalCase → ['pascal', 'case'] + * - snake_case → ['snake', 'case'] + * - kebab-case → ['kebab', 'case'] + * - dot.notation → ['dot', 'notation'] + * - File paths → individual path components (no extension) + * + * @param {string} text + * @param {object} [opts] + * @param {boolean} [opts.removeStopWords=true] + * @param {number} [opts.minLength=2] + * @returns {string[]} + */ +function tokenize(text, opts) { + if (!text || typeof text !== 'string') return []; + const removeStop = opts && opts.removeStopWords === false ? false : true; + const minLen = (opts && opts.minLength) || 2; + + const tokens = text + // strip file extension (e.g. .js, .ts, .py) + .replace(/\.\w{1,6}(?=\s|\/|$)/g, ' ') + // camelCase / PascalCase split + .replace(/([a-z])([A-Z])/g, '$1 $2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + // snake_case / kebab-case / dot.notation + .replace(/[_\-\.\/]/g, ' ') + // drop remaining non-word characters + .replace(/[^\w\s]/g, ' ') + .toLowerCase() + .split(/\s+/) + .filter((t) => t.length >= minLen); + + if (!removeStop) return [...new Set(tokens)]; + return [...new Set(tokens.filter((t) => !STOP_WORDS.has(t)))]; +} + +module.exports = { tokenize, STOP_WORDS }; diff --git a/test/integration/analyze.test.js b/test/integration/analyze.test.js index 036f8728..b3e43cd3 100644 --- a/test/integration/analyze.test.js +++ b/test/integration/analyze.test.js @@ -17,7 +17,7 @@ * 11. CLI --analyze --json produces valid JSON with correct keys * 12. CLI --analyze --slow runs without throw * 13. CLI --diagnose-extractors exits 0 and prints pass/fail counts - * 14. CLI --version returns 2.2.0 + * 14. CLI --version returns 2.3.0 */ const assert = require('assert'); @@ -224,16 +224,16 @@ test('CLI --diagnose-extractors: exits 0 and shows results', () => { }); // --------------------------------------------------------------------------- -// 14. CLI --version returns 2.2.0 +// 14. CLI --version returns 2.3.0 // --------------------------------------------------------------------------- -test('CLI --version: returns 2.2.0', () => { +test('CLI --version: returns 2.3.0', () => { const r = spawnSync(process.execPath, [SCRIPT, '--version'], { cwd: ROOT, encoding: 'utf8', timeout: 10000, }); assert.strictEqual(r.status, 0); - assert.ok(r.stdout.trim().includes('2.2.0'), `got: ${r.stdout.trim()}`); + assert.ok(r.stdout.trim().includes('2.3.0'), `got: ${r.stdout.trim()}`); }); // --------------------------------------------------------------------------- diff --git a/test/integration/mcp-server.test.js b/test/integration/mcp-server.test.js index ebaddcbb..781b770c 100644 --- a/test/integration/mcp-server.test.js +++ b/test/integration/mcp-server.test.js @@ -99,14 +99,14 @@ test('initialize returns serverInfo', () => { }); // ───────────────────────────────────────────────────────────── -// Gate 2: tools/list returns 7 tools (v1.4+) +// Gate 2: tools/list returns 8 tools (v2.3+) // ───────────────────────────────────────────────────────────── test('tools/list returns exactly 7 tools', () => { withTempProject((dir) => { const [res] = mcpCall({ jsonrpc: '2.0', method: 'tools/list', id: 2 }, dir); assert.ok(res.result, 'Should have result'); assert.ok(Array.isArray(res.result.tools), 'tools should be array'); - assert.strictEqual(res.result.tools.length, 7); + assert.strictEqual(res.result.tools.length, 8); const names = res.result.tools.map((t) => t.name); assert.ok(names.includes('read_context'), 'Should have read_context'); assert.ok(names.includes('search_signatures'), 'Should have search_signatures'); @@ -115,6 +115,7 @@ test('tools/list returns exactly 7 tools', () => { assert.ok(names.includes('get_routing'), 'Should have get_routing'); assert.ok(names.includes('explain_file'), 'Should have explain_file'); assert.ok(names.includes('list_modules'), 'Should have list_modules'); + assert.ok(names.includes('query_context'), 'Should have query_context'); }); }); diff --git a/test/integration/mcp-v14.test.js b/test/integration/mcp-v14.test.js index b90608a7..3a0758c4 100644 --- a/test/integration/mcp-v14.test.js +++ b/test/integration/mcp-v14.test.js @@ -78,7 +78,7 @@ function seedContextFile(dir) { } // ───────────────────────────────────────────────────────────── -// Gate: tools/list now returns 7 tools including v1.4 additions +// Gate: tools/list now returns 8 tools including v2.3 query_context // ───────────────────────────────────────────────────────────── console.log('\nMCP v1.4 — tools/list\n'); @@ -88,10 +88,11 @@ test('tools/list returns exactly 7 tools', () => { const [res] = mcpCall({ jsonrpc: '2.0', method: 'tools/list', id: 1 }, dir); assert.ok(res.result, 'Should have result'); assert.ok(Array.isArray(res.result.tools), 'tools should be array'); - assert.strictEqual(res.result.tools.length, 7, `Expected 7 tools, got ${res.result.tools.length}`); + assert.strictEqual(res.result.tools.length, 8, `Expected 8 tools, got ${res.result.tools.length}`); const names = res.result.tools.map((t) => t.name); assert.ok(names.includes('explain_file'), 'Should have explain_file'); assert.ok(names.includes('list_modules'), 'Should have list_modules'); + assert.ok(names.includes('query_context'), 'Should have query_context'); }); }); diff --git a/test/integration/retrieval.test.js b/test/integration/retrieval.test.js new file mode 100644 index 00000000..9b080533 --- /dev/null +++ b/test/integration/retrieval.test.js @@ -0,0 +1,315 @@ +'use strict'; + +/** + * Integration tests for v2.3 query-aware retrieval. + * + * Tests: + * 1. tokenize: splits camelCase into tokens + * 2. tokenize: splits snake_case into tokens + * 3. tokenize: removes stop words by default + * 4. tokenize: keeps stop words when removeStopWords=false + * 5. tokenize: handles file path input + * 6. tokenize: returns empty array for empty input + * 7. rank: returns sorted array for a valid query + * 8. rank: score is a non-negative number + * 9. rank: topK limits result count + * 10. rank: empty query returns top-K by sig count + * 11. rank: returns empty array for empty sigIndex + * 12. rank: python extractor file in top-3 for "python extractor" query + * 13. formatRankTable: output contains query header and columns + * 14. formatRankJSON: has correct top-level keys + * 15. CLI --query: exits 0 and prints ranked table + * 16. CLI --query --json: valid JSON with correct keys + * 17. CLI --query --top 3: returns at most 3 results + * 18. CLI --query missing arg: exits 1 with usage message + * 19. CLI --version: returns 2.3.0 + * 20. MCP tools/list: returns 8 tools including query_context + * 21. MCP query_context: returns result for a valid query + * 22. MCP query_context: returns error for missing query arg + * 23. MCP query_context: unknown tool still returns error + */ + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const ROOT = path.resolve(__dirname, '../..'); +const SCRIPT = path.join(ROOT, 'gen-context.js'); + +let passed = 0; +let failed = 0; + +function test(name, fn) { + try { + fn(); + console.log(` PASS ${name}`); + passed++; + } catch (err) { + console.log(` FAIL ${name}: ${err.message}`); + failed++; + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- +function run(...args) { + return spawnSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, + encoding: 'utf8', + timeout: 15000, + maxBuffer: 2 * 1024 * 1024, + }); +} + +function mcpCall(msg, cwd) { + const res = spawnSync(process.execPath, [SCRIPT, '--mcp'], { + input: JSON.stringify(msg) + '\n', + cwd: cwd || ROOT, + encoding: 'utf8', + timeout: 10000, + maxBuffer: 1024 * 1024, + }); + return res.stdout.trim().split('\n').filter(Boolean).map((l) => JSON.parse(l)); +} + +// --------------------------------------------------------------------------- +// Load modules directly from src/ +// --------------------------------------------------------------------------- +const { tokenize } = require(path.join(ROOT, 'src', 'retrieval', 'tokenizer')); +const { rank, buildSigIndex, formatRankTable, formatRankJSON } = + require(path.join(ROOT, 'src', 'retrieval', 'ranker')); + +// Build a minimal sig index from SigMap's own context file +const sigIndex = buildSigIndex(ROOT); + +console.log('[retrieval.test.js] v2.3 query-aware retrieval'); +console.log(''); + +// --------------------------------------------------------------------------- +// tokenize — unit tests +// --------------------------------------------------------------------------- +test('tokenize: splits camelCase into tokens', () => { + const tokens = tokenize('analyzeFiles'); + assert.ok(tokens.includes('analyze'), `expected "analyze" in ${tokens}`); + assert.ok(tokens.includes('files'), `expected "files" in ${tokens}`); +}); + +test('tokenize: splits snake_case into tokens', () => { + const tokens = tokenize('build_sig_index'); + assert.ok(tokens.includes('build'), `expected "build" in ${tokens}`); + assert.ok(tokens.includes('sig'), `expected "sig" in ${tokens}`); + assert.ok(tokens.includes('index'), `expected "index" in ${tokens}`); +}); + +test('tokenize: removes stop words by default', () => { + const tokens = tokenize('the function in a module'); + assert.ok(!tokens.includes('the'), 'should remove "the"'); + assert.ok(!tokens.includes('a'), 'should remove "a"'); + assert.ok(!tokens.includes('in'), 'should remove "in"'); + assert.ok(tokens.includes('function'), 'should keep "function"'); + assert.ok(tokens.includes('module'), 'should keep "module"'); +}); + +test('tokenize: keeps stop words when removeStopWords=false', () => { + const tokens = tokenize('the function', { removeStopWords: false }); + assert.ok(tokens.includes('the'), 'should keep "the"'); +}); + +test('tokenize: handles file path input', () => { + const tokens = tokenize('src/extractors/python.js'); + assert.ok(tokens.includes('src'), `expected "src" in ${tokens}`); + assert.ok(tokens.includes('extractors'), `expected "extractors" in ${tokens}`); + assert.ok(tokens.includes('python'), `expected "python" in ${tokens}`); +}); + +test('tokenize: returns empty array for empty input', () => { + assert.deepStrictEqual(tokenize(''), []); + assert.deepStrictEqual(tokenize(null), []); + assert.deepStrictEqual(tokenize(undefined), []); +}); + +// --------------------------------------------------------------------------- +// rank — unit tests (using SigMap's own sig index) +// --------------------------------------------------------------------------- +test('rank: returns sorted array for a valid query', () => { + if (sigIndex.size === 0) { /* skip if no context file generated yet */ return; } + const results = rank('python extractor', sigIndex, { topK: 5 }); + assert.ok(Array.isArray(results), 'should return array'); + assert.ok(results.length > 0, 'should return at least one result'); + // Verify descending sort by score + for (let i = 1; i < results.length; i++) { + assert.ok(results[i].score <= results[i - 1].score, 'results should be sorted desc by score'); + } +}); + +test('rank: score is a non-negative number', () => { + if (sigIndex.size === 0) return; + const results = rank('extract', sigIndex, { topK: 3 }); + for (const r of results) { + assert.strictEqual(typeof r.score, 'number'); + assert.ok(r.score >= 0, `score should be non-negative, got ${r.score}`); + } +}); + +test('rank: topK limits result count', () => { + if (sigIndex.size === 0) return; + const limit = 3; + const results = rank('extractor', sigIndex, { topK: limit }); + assert.ok(results.length <= limit, `expected ≤ ${limit} results, got ${results.length}`); +}); + +test('rank: empty query returns top-K by sig count', () => { + if (sigIndex.size === 0) return; + const results = rank('', sigIndex, { topK: 5 }); + assert.ok(Array.isArray(results)); + assert.ok(results.length <= 5); + // Each result must have the required shape + for (const r of results) { + assert.ok('file' in r); + assert.ok('score' in r); + assert.ok('sigs' in r); + assert.ok('tokens' in r); + } +}); + +test('rank: returns empty array for empty sigIndex', () => { + const empty = new Map(); + const results = rank('anything', empty, { topK: 5 }); + assert.deepStrictEqual(results, []); +}); + +test('rank: python extractor file in top-3 for "python extractor" query', () => { + if (sigIndex.size === 0) return; // no context file — skip, don't fail + const results = rank('python extractor', sigIndex, { topK: 10 }); + const top3 = results.slice(0, 3).map((r) => r.file); + const hasPython = top3.some((f) => f.includes('python')); + assert.ok(hasPython, `expected python extractor in top 3, got: ${top3.join(', ')}`); +}); + +// --------------------------------------------------------------------------- +// formatRankTable / formatRankJSON — unit tests +// --------------------------------------------------------------------------- +test('formatRankTable: output contains query header and columns', () => { + if (sigIndex.size === 0) return; + const results = rank('scanner', sigIndex, { topK: 3 }); + const table = formatRankTable(results, 'scanner'); + assert.ok(table.includes('scanner'), 'should include query'); + assert.ok(table.includes('Rank'), 'should include Rank column'); + assert.ok(table.includes('File'), 'should include File column'); + assert.ok(table.includes('Score'), 'should include Score column'); +}); + +test('formatRankJSON: has correct top-level keys', () => { + if (sigIndex.size === 0) return; + const results = rank('route', sigIndex, { topK: 3 }); + const obj = formatRankJSON(results, 'route'); + assert.ok('query' in obj, 'should have query'); + assert.ok('results' in obj, 'should have results'); + assert.ok('totalResults' in obj, 'should have totalResults'); + assert.ok(Array.isArray(obj.results), 'results should be array'); + for (const r of obj.results) { + assert.ok('rank' in r, 'each result should have rank'); + assert.ok('file' in r, 'each result should have file'); + assert.ok('score' in r, 'each result should have score'); + assert.ok('sigs' in r, 'each result should have sigs'); + assert.ok('tokens' in r, 'each result should have tokens'); + } +}); + +// --------------------------------------------------------------------------- +// CLI tests +// --------------------------------------------------------------------------- +test('CLI --query: exits 0 and prints ranked table', () => { + // Use --query "extract" — something that should hit extractors + const res = run('--query', 'extract'); + assert.strictEqual(res.status, 0, `Expected exit 0, got ${res.status}. stderr: ${res.stderr}`); + const out = res.stdout + res.stderr; + // Output should contain either a Rank table or the no-match message + assert.ok(out.length > 0, 'should produce output'); +}); + +test('CLI --query --json: valid JSON with correct keys', () => { + const res = run('--query', 'python extractor', '--json'); + assert.strictEqual(res.status, 0, `Expected exit 0, got ${res.status}. stderr: ${res.stderr}`); + let obj; + try { + obj = JSON.parse(res.stdout.trim()); + } catch (e) { + assert.fail(`Output is not valid JSON: ${res.stdout.slice(0, 200)}`); + } + assert.ok('query' in obj, 'should have query'); + assert.ok('results' in obj, 'should have results'); + assert.ok('totalResults' in obj, 'should have totalResults'); + assert.ok(Array.isArray(obj.results), 'results should be array'); +}); + +test('CLI --query --top 3: returns at most 3 results', () => { + const res = run('--query', 'extractor', '--json', '--top', '3'); + assert.strictEqual(res.status, 0, `Expected exit 0. stderr: ${res.stderr}`); + const obj = JSON.parse(res.stdout.trim()); + assert.ok(obj.results.length <= 3, `Expected ≤ 3 results, got ${obj.results.length}`); +}); + +test('CLI --query missing arg: exits 1 with usage message', () => { + const res = run('--query'); + assert.strictEqual(res.status, 1, `Expected exit 1, got ${res.status}`); + const out = res.stdout + res.stderr; + assert.ok(out.includes('--query'), 'should mention --query in error'); +}); + +test('CLI --version: returns 2.3.0', () => { + const res = run('--version'); + assert.strictEqual(res.status, 0); + assert.ok(res.stdout.trim().includes('2.3.0'), `expected 2.3.0, got: ${res.stdout.trim()}`); +}); + +// --------------------------------------------------------------------------- +// MCP tests — query_context (8th tool) +// --------------------------------------------------------------------------- +test('MCP tools/list: returns 8 tools including query_context', () => { + const [res] = mcpCall({ jsonrpc: '2.0', method: 'tools/list', id: 1 }); + assert.ok(res.result, 'should have result'); + assert.strictEqual(res.result.tools.length, 8, `expected 8 tools, got ${res.result.tools.length}`); + const names = res.result.tools.map((t) => t.name); + assert.ok(names.includes('query_context'), 'should include query_context'); +}); + +test('MCP query_context: returns result for a valid query', () => { + const [res] = mcpCall({ + jsonrpc: '2.0', method: 'tools/call', id: 2, + params: { name: 'query_context', arguments: { query: 'extractor', topK: 5 } }, + }); + assert.ok(res.result, 'should have result'); + const text = res.result.content[0].text; + assert.ok(typeof text === 'string', 'should return string'); + assert.ok(text.length > 0, 'should return non-empty output'); +}); + +test('MCP query_context: returns error for missing query arg', () => { + const [res] = mcpCall({ + jsonrpc: '2.0', method: 'tools/call', id: 3, + params: { name: 'query_context', arguments: {} }, + }); + assert.ok(res.result, 'should have result'); + const text = res.result.content[0].text; + assert.ok(text.toLowerCase().includes('missing') || text.toLowerCase().includes('required'), + `expected error message, got: ${text}`); +}); + +test('MCP query_context: unknown tool still returns error', () => { + const [res] = mcpCall({ + jsonrpc: '2.0', method: 'tools/call', id: 4, + params: { name: 'nonexistent_tool', arguments: {} }, + }); + assert.ok(res.error || (res.result && res.result.content), 'should get error or result'); +}); + +// --------------------------------------------------------------------------- +// Results +// --------------------------------------------------------------------------- +console.log(''); +console.log(`${passed} passed, ${failed} failed`); + +if (failed > 0) process.exit(1);