From 53c1f21b2013e93c626db1d414df76999aec5b08 Mon Sep 17 00:00:00 2001 From: Mufacoderz Date: Sun, 14 Jun 2026 16:13:29 +0800 Subject: [PATCH] feat: add command-based model routing Co-authored-by: devmap-agent <238585242+devmap-agent@users.noreply.github.com> --- PRD.md | 24 ++++++-- docs/architecture.md | 9 ++- docs/commands.md | 44 +++++++++++--- docs/for-me-personal/PROGRESS.md | 10 +++ docs/for-me-personal/TEST.md | 26 ++++++++ packages/cli/src/ai/groq.ts | 6 +- packages/cli/src/commands/config.ts | 43 +++++++++++++ packages/cli/src/index.ts | 9 +++ packages/cli/src/utils/help.ts | 1 + packages/cli/test/analyze-ai.test.ts | 51 ++++++++++++++++ packages/cli/test/config-command.test.ts | 77 ++++++++++++++++++++++++ packages/cli/test/doctor.test.ts | 2 +- 12 files changed, 281 insertions(+), 21 deletions(-) create mode 100644 packages/cli/src/commands/config.ts create mode 100644 packages/cli/test/config-command.test.ts diff --git a/PRD.md b/PRD.md index be5fbc7..13e8c6c 100644 --- a/PRD.md +++ b/PRD.md @@ -251,7 +251,7 @@ MVP success means DevMap can: ## 9. Core Commands — MVP -Four commands. No more, no less. +Four core project commands plus one configuration command. > Nothing removed. Nothing added until MVP is shipped. @@ -422,6 +422,21 @@ No issues found. --- +### `devmap config model` + +Set a global Groq model override or restore automatic command-based routing. + +```bash +devmap config model llama-3.1-8b-instant +devmap config model openai/gpt-oss-120b +devmap config model auto +``` + +The override applies to AI-powered commands. `auto` restores the defaults in +the model routing table. + +--- + ## 10. Generated Files DevMap uses generated files to create reusable context for humans and AI agents. @@ -538,9 +553,10 @@ ai/ | Command | Model | Reason | |---|---|---| -| `analyze`, `ask` | `openai/gpt-oss-20b` | Production model, fast and cost-efficient | -| `analyze --deep` | `llama-3.3-70b-versatile` | Better reasoning for detailed explanations | -| Fallback | `llama-3.3-70b-versatile` | Production fallback when primary model is unavailable | +| `ask` | `llama-3.1-8b-instant` | Fast model for focused codebase questions | +| `analyze` | `openai/gpt-oss-20b` | Balanced architecture interpretation | +| `analyze --deep` | `openai/gpt-oss-120b` | Heavy cross-module reasoning | +| Fallback | `openai/gpt-oss-20b` | Production fallback when a different primary model is unavailable | If a model becomes unavailable, DevMap gracefully falls back. No raw provider errors shown to users. diff --git a/docs/architecture.md b/docs/architecture.md index b3332a8..5e639b7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -491,13 +491,16 @@ MVP default model routing: | Command | Model | | ---------------- | ------------------------- | +| `ask` | `llama-3.1-8b-instant` | | `analyze` | `openai/gpt-oss-20b` | -| `ask` | `openai/gpt-oss-20b` | -| `analyze --deep` | `llama-3.3-70b-versatile` | -| Fallback | `llama-3.3-70b-versatile` | +| `analyze --deep` | `openai/gpt-oss-120b` | +| Fallback | `openai/gpt-oss-20b` | If a model is unavailable, DevMap should fall back gracefully. Only Groq production models should be used as public defaults. + +Users can override automatic routing with `devmap config model `. +Running `devmap config model auto` restores the defaults above. Raw provider errors should not be shown directly to users. diff --git a/docs/commands.md b/docs/commands.md index ceb9747..af9d10c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -6,14 +6,15 @@ ## Overview -DevMap MVP provides four core commands: +DevMap MVP provides four core project commands and one configuration command: * `devmap init` * `devmap analyze` * `devmap ask` -* `devmap doctor` - -No additional commands should be added until the MVP is shipped. +* `devmap doctor` +* `devmap config model` + +No additional product commands should be added until the MVP is shipped. Future commands are documented in: @@ -366,7 +367,7 @@ Use existing snapshot or re-analyze first? --- -## `devmap doctor` +## `devmap doctor` Run diagnostics for DevMap setup. @@ -435,11 +436,34 @@ Issues found: * Output must be readable * Errors must be actionable * Do not expose raw stack traces -* Mention what command the user should run next - ---- - -## Global Flags +* Mention what command the user should run next + +--- + +## `devmap config model` + +Set a global model override for AI-powered commands. + +### Usage + +```bash +devmap config model llama-3.1-8b-instant +devmap config model openai/gpt-oss-120b +devmap config model auto +``` + +`auto` restores command-based routing: + +* `ask` uses `llama-3.1-8b-instant` +* `analyze` uses `openai/gpt-oss-20b` +* `analyze --deep` uses `openai/gpt-oss-120b` + +The command preserves the configured provider and API key. DevMap must be +initialized before changing the model. + +--- + +## Global Flags Available globally: diff --git a/docs/for-me-personal/PROGRESS.md b/docs/for-me-personal/PROGRESS.md index b270d5f..9e1a976 100644 --- a/docs/for-me-personal/PROGRESS.md +++ b/docs/for-me-personal/PROGRESS.md @@ -4,6 +4,16 @@ Terakhir diperbarui: 2026-06-14 ## Update 2026-06-14 +### Model Routing And Config + +- Default `devmap ask` memakai `llama-3.1-8b-instant`. +- Standard `devmap analyze` tetap memakai `openai/gpt-oss-20b`. +- `devmap analyze --deep` memakai `openai/gpt-oss-120b`. +- Fallback model memakai `openai/gpt-oss-20b`. +- Command `devmap config model ` dapat menetapkan override global tanpa + mengubah provider atau API key. +- `devmap config model auto` mengembalikan routing default per command. + ### Context Builder Token Optimization - Pertanyaan navigasi English seperti `where` dan `find` sekarang memakai diff --git a/docs/for-me-personal/TEST.md b/docs/for-me-personal/TEST.md index 61f8487..fb29f4f 100644 --- a/docs/for-me-personal/TEST.md +++ b/docs/for-me-personal/TEST.md @@ -44,6 +44,32 @@ Periksa `Relevant Files` dan prompt token usage. Query pertama seharusnya memprioritaskan production CLI source dan memakai context jauh lebih kecil daripada default lama lima file dengan maksimal 200 baris per file. +## Model Routing And Override + +Focused automated test: + +```powershell +pnpm --filter devmap exec tsx --test test/config-command.test.ts test/analyze-ai.test.ts test/ask-command.test.ts +``` + +Expected automatic routing: + +- `ask`: `llama-3.1-8b-instant` +- `analyze`: `openai/gpt-oss-20b` +- `analyze --deep`: `openai/gpt-oss-120b` +- fallback: `openai/gpt-oss-20b` + +Manual override: + +```powershell +pnpm dev:cli config model openai/gpt-oss-120b +pnpm dev:cli doctor +pnpm dev:cli config model auto +``` + +The first command should preserve the existing API key and provider. The last +command should restore automatic command-based routing. + ## Urutan Testing Yang Direkomendasikan Untuk development harian: diff --git a/packages/cli/src/ai/groq.ts b/packages/cli/src/ai/groq.ts index 7a0f9d2..fd694af 100644 --- a/packages/cli/src/ai/groq.ts +++ b/packages/cli/src/ai/groq.ts @@ -13,10 +13,10 @@ const MAX_RATE_LIMIT_DELAY_MS = 10_000; const MAX_RATE_LIMIT_RETRIES = 3; export const DEFAULT_AI_MODELS = { - ask: "openai/gpt-oss-20b", + ask: "llama-3.1-8b-instant", analyze: "openai/gpt-oss-20b", - deepAnalyze: "llama-3.3-70b-versatile", - fallback: "llama-3.3-70b-versatile" + deepAnalyze: "openai/gpt-oss-120b", + fallback: "openai/gpt-oss-20b" } as const; export type GroqClientDependencies = { diff --git a/packages/cli/src/commands/config.ts b/packages/cli/src/commands/config.ts new file mode 100644 index 0000000..ea61cb7 --- /dev/null +++ b/packages/cli/src/commands/config.ts @@ -0,0 +1,43 @@ +import { + readConfig, + writeConfig, + type DevmapConfig +} from "../utils/config.js"; +import { output } from "../utils/output.js"; + +export type ConfigDependencies = { + loadConfig?: () => Promise; + persistConfig?: (config: DevmapConfig) => Promise; +}; + +export async function configModelCommand( + model: string, + dependencies: ConfigDependencies = {} +): Promise { + const selectedModel = model.trim(); + if (!selectedModel) { + output.error("Model name cannot be empty."); + return; + } + + const loadConfig = dependencies.loadConfig ?? readConfig; + const persistConfig = dependencies.persistConfig ?? writeConfig; + const config = await loadConfig(); + + if (!config) { + output.error("DevMap is not configured yet."); + output.note("Run devmap init before changing the model."); + return; + } + + await persistConfig({ + ...config, + model: selectedModel + }); + + output.success( + selectedModel === "auto" + ? "Restored automatic command-based model routing." + : `Default model override set to ${selectedModel}.` + ); +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 4411740..d18ba5f 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -2,6 +2,7 @@ import { Command } from "commander"; import { analyzeCommand } from "./commands/analyze.js"; import { askCommand } from "./commands/ask.js"; +import { configModelCommand } from "./commands/config.js"; import { doctorCommand } from "./commands/doctor.js"; import { initCommand } from "./commands/init.js"; import { printHelp } from "./utils/help.js"; @@ -34,6 +35,14 @@ program .argument("", "question to ask") .action(askCommand); +program + .command("config") + .description("Update DevMap configuration") + .command("model") + .description("Set a model override or restore automatic routing") + .argument("", "Groq model ID or auto") + .action(configModelCommand); + program .command("doctor") .description("Diagnose DevMap setup") diff --git a/packages/cli/src/utils/help.ts b/packages/cli/src/utils/help.ts index 867913f..1e2030d 100644 --- a/packages/cli/src/utils/help.ts +++ b/packages/cli/src/utils/help.ts @@ -4,6 +4,7 @@ const commands = [ ["init", "Initialize DevMap configuration"], ["analyze", "Analyze project structure"], ["ask ", "Ask about your codebase"], + ["config model", "Set model override or automatic routing"], ["doctor", "Diagnose DevMap setup"] ] as const; diff --git a/packages/cli/test/analyze-ai.test.ts b/packages/cli/test/analyze-ai.test.ts index 96a8844..be9f8d6 100644 --- a/packages/cli/test/analyze-ai.test.ts +++ b/packages/cli/test/analyze-ai.test.ts @@ -3,6 +3,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; +import { DEFAULT_AI_MODELS } from "../src/ai/groq.js"; import type { AiClient, AiCompletionRequest, @@ -124,6 +125,56 @@ test("analyze warns and continues when package.json is malformed", async () => { } }); +test("analyze auto routing uses 20B normally and 120B for deep analysis", async () => { + const projectRoot = await mkdtemp(join(tmpdir(), "devmap-model-routing-")); + const requests: AiCompletionRequest[] = []; + const client: AiClient = { + async complete(request): Promise { + requests.push(request); + return { + content: "Architecture result.", + model: request.model + }; + } + }; + + try { + await writeFile( + join(projectRoot, "package.json"), + JSON.stringify({ name: "model-routing-fixture" }), + "utf8" + ); + await writeFile(join(projectRoot, "index.ts"), "export const ready = true;\n", "utf8"); + + const dependencies = { + loadConfig: async () => ({ + provider: "groq" as const, + apiKey: "gsk_fixture", + model: "auto" + }), + createAiClient: () => client + }; + + await captureOutput(() => analyzeCommand( + projectRoot, + { fresh: true }, + dependencies + )); + await captureOutput(() => analyzeCommand( + projectRoot, + { deep: true, fresh: true }, + dependencies + )); + + assert.equal(requests[0]?.model, DEFAULT_AI_MODELS.analyze); + assert.equal(requests[1]?.model, DEFAULT_AI_MODELS.deepAnalyze); + assert.equal(requests[0]?.fallbackModel, DEFAULT_AI_MODELS.fallback); + assert.equal(requests[1]?.fallbackModel, DEFAULT_AI_MODELS.fallback); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } +}); + function stripAnsi(value: string): string { return value.replace(/\u001B\[[0-9;]*m/g, ""); } diff --git a/packages/cli/test/config-command.test.ts b/packages/cli/test/config-command.test.ts new file mode 100644 index 0000000..0dd83f6 --- /dev/null +++ b/packages/cli/test/config-command.test.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { configModelCommand } from "../src/commands/config.js"; +import type { DevmapConfig } from "../src/utils/config.js"; + +test("config model updates the model while preserving Groq credentials", async () => { + const current: DevmapConfig = { + provider: "groq", + apiKey: "gsk_fixture", + model: "auto" + }; + let saved: DevmapConfig | null = null; + + await configModelCommand("openai/gpt-oss-120b", { + loadConfig: async () => current, + persistConfig: async (config) => { + saved = config; + } + }); + + assert.deepEqual(saved, { + provider: "groq", + apiKey: "gsk_fixture", + model: "openai/gpt-oss-120b" + }); +}); + +test("config model accepts auto to restore command-based routing", async () => { + const current: DevmapConfig = { + provider: "groq", + apiKey: "gsk_fixture", + model: "openai/gpt-oss-120b" + }; + let saved: DevmapConfig | null = null; + + await configModelCommand("auto", { + loadConfig: async () => current, + persistConfig: async (config) => { + saved = config; + } + }); + + assert.equal(saved?.model, "auto"); +}); + +test("config model requires an existing initialized config", async () => { + let persisted = false; + const logs = await captureOutput(() => configModelCommand( + "llama-3.1-8b-instant", + { + loadConfig: async () => null, + persistConfig: async () => { + persisted = true; + } + } + )); + + assert.equal(persisted, false); + assert.match(logs, /Run devmap init/i); +}); + +async function captureOutput(action: () => Promise): Promise { + const logs: string[] = []; + const originalLog = console.log; + const originalError = console.error; + + console.log = (...values: unknown[]) => logs.push(values.join(" ")); + console.error = (...values: unknown[]) => logs.push(values.join(" ")); + + try { + await action(); + return logs.join("\n"); + } finally { + console.log = originalLog; + console.error = originalError; + } +} diff --git a/packages/cli/test/doctor.test.ts b/packages/cli/test/doctor.test.ts index 42bcaf5..51e9083 100644 --- a/packages/cli/test/doctor.test.ts +++ b/packages/cli/test/doctor.test.ts @@ -41,7 +41,7 @@ test("doctor reports project, provider, model, and snapshot diagnostics", async assert.match(logs, /Framework\s+express/); assert.match(logs, /Provider\s+groq/); assert.match(logs, /API key\s+valid/); - assert.match(logs, /Model\s+openai\/gpt-oss-20b/); + assert.match(logs, /Model\s+llama-3\.1-8b-instant/); assert.match(logs, /Snapshot\s+valid/); assert.match(logs, /No issues found/); assert.doesNotMatch(logs, /gsk_fixture/);