Skip to content

Commit 5085167

Browse files
seanhancaclaude
andcommitted
feat: intent pipeline coverage — 46 tests, 2 gaps closed
Tests cover: compare_models (explicit + auto-fill + N models), batch_generate, variations, style_sweep, story, single (false-positive guards), edge cases, cleanPrompt, extractMentionedModels. Gaps found and fixed: 1. "5 models including recraft and nano" — named models (2) triggered before requested count (5). Fix: use max(named, requested) as target count. 2. Batch regex missed last item after "and" — e.g. "a cat, a dog, a bird, and a goldfish" found 3 not 4. Fix: added $ anchor to regex lookahead. Also: detectModelCountRequest now handles "compare models" (no count) and "different models" keyword. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 41c8a55 commit 5085167

2 files changed

Lines changed: 405 additions & 43 deletions

File tree

packages/creative-kit/src/agent/intent-planner.ts

Lines changed: 36 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -240,11 +240,14 @@ export function autoFillModels(named: string[], requested: number): string[] {
240240
*/
241241
function detectModelCountRequest(text: string): number {
242242
const lower = text.toLowerCase();
243-
// "4 different models", "using 3 models", "compare 5 models"
243+
// "4 different models", "using 3 models", "compare 5 models",
244+
// "with 5 different models", "compare with 4 models"
244245
const m = lower.match(/(\d+)\s+(?:different\s+)?(?:models?|ai\s+models?)/);
245246
if (m) return parseInt(m[1], 10);
246247
// "multiple models", "several models", "various models"
247-
if (/(?:multiple|several|various|many|all)\s+(?:different\s+)?(?:models?|ai\s+models?)/.test(lower)) return 4;
248+
if (/(?:multiple|several|various|many|all|different)\s+(?:different\s+)?(?:models?|ai\s+models?)/.test(lower)) return 4;
249+
// "compare models" without a count
250+
if (/compare\s+(?:the\s+)?(?:models?|ai\s+models?)/.test(lower)) return 4;
248251
return 0;
249252
}
250253

@@ -253,28 +256,22 @@ function detectModelCountRequest(text: string): number {
253256
export function classifyWithRegex(text: string): IntentPlan {
254257
const lower = text.toLowerCase();
255258
const models = extractMentionedModels(text);
256-
257-
// Compare models: 2+ model names explicitly
258-
if (models.length >= 2) {
259-
return {
260-
type: "compare_models",
261-
confidence: 0.95,
262-
models,
263-
prompt: cleanPrompt(text),
264-
reason: `Detected ${models.length} model names`,
265-
};
266-
}
267-
268-
// Compare models: "N models" / "N different models" (partially named)
269259
const requestedCount = detectModelCountRequest(text);
270-
if (requestedCount >= 2) {
271-
const filled = autoFillModels(models, requestedCount);
260+
const targetCount = Math.max(models.length, requestedCount);
261+
262+
// Compare models: 2+ model names or "N models" request
263+
if (targetCount >= 2) {
264+
const finalModels = targetCount > models.length
265+
? autoFillModels(models, targetCount)
266+
: models;
272267
return {
273268
type: "compare_models",
274-
confidence: 0.85,
275-
models: filled,
269+
confidence: targetCount > models.length ? 0.85 : 0.95,
270+
models: finalModels,
276271
prompt: cleanPrompt(text),
277-
reason: `User requested ${requestedCount} models (${models.length} named, ${filled.length - models.length} auto-filled): ${filled.join(", ")}`,
272+
reason: targetCount > models.length
273+
? `User requested ${requestedCount} models (${models.length} named, auto-filled to ${finalModels.length}): ${finalModels.join(", ")}`
274+
: `Detected ${models.length} model names: ${models.join(", ")}`,
278275
};
279276
}
280277

@@ -305,7 +302,8 @@ export function classifyWithRegex(text: string): IntentPlan {
305302
}
306303

307304
// Batch: "make a X, a Y, and a Z" (distinct articles/items)
308-
const articleItems = text.match(/\ba\s+\w+(?:\s+\w+){0,3}(?=\s*[,]|\s+and\s)/gi);
305+
// Match "a <words>" followed by comma, "and", or end of sentence
306+
const articleItems = text.match(/\ba\s+\w+(?:\s+\w+){0,3}(?=\s*[,.]|\s+and\s|$)/gi);
309307
if (articleItems && articleItems.length >= 3) {
310308
return {
311309
type: "batch_generate",
@@ -353,34 +351,29 @@ export async function planIntent(
353351
config?: LLMClassifierConfig,
354352
): Promise<IntentPlan> {
355353
// Tier 0: Deterministic checks — ALWAYS run first.
356-
// Model names are facts. If 2+ model names appear, it's a comparison.
357354
const models = extractMentionedModels(text);
358-
if (models.length >= 2) {
359-
const plan: IntentPlan = {
360-
type: "compare_models",
361-
confidence: 0.99,
362-
models,
363-
prompt: cleanPrompt(text),
364-
reason: `${models.length} model names detected: ${models.join(", ")}`,
365-
};
366-
console.log(`[IntentPlanner] Deterministic: ${plan.reason}`);
367-
return plan;
368-
}
369-
370-
// "N different models" / "4 models" — user wants comparison but named
371-
// fewer than N. Auto-fill from defaults (e.g. "4 models, include gpt-image"
372-
// → gpt-image + flux-dev + recraft-v4 + nano-banana)
373355
const requestedCount = detectModelCountRequest(text);
374-
if (requestedCount >= 2) {
375-
const filled = autoFillModels(models, requestedCount);
356+
357+
// User explicitly requested N models (may have named some or all)
358+
// Use the LARGER of: named model count vs requested count
359+
// "5 different models including recraft and nano" → 5 (not 2)
360+
const targetCount = Math.max(models.length, requestedCount);
361+
362+
if (targetCount >= 2) {
363+
const finalModels = targetCount > models.length
364+
? autoFillModels(models, targetCount)
365+
: models;
366+
const isAutoFilled = targetCount > models.length;
376367
const plan: IntentPlan = {
377368
type: "compare_models",
378-
confidence: 0.90,
379-
models: filled,
369+
confidence: isAutoFilled ? 0.90 : 0.99,
370+
models: finalModels,
380371
prompt: cleanPrompt(text),
381-
reason: `User requested ${requestedCount} models (${models.length} named, auto-filled to ${filled.length}): ${filled.join(", ")}`,
372+
reason: isAutoFilled
373+
? `User requested ${requestedCount} models (${models.length} named, auto-filled to ${finalModels.length}): ${finalModels.join(", ")}`
374+
: `${models.length} model names detected: ${models.join(", ")}`,
382375
};
383-
console.log(`[IntentPlanner] Deterministic (auto-fill): ${plan.reason}`);
376+
console.log(`[IntentPlanner] Deterministic${isAutoFilled ? " (auto-fill)" : ""}: ${plan.reason}`);
384377
return plan;
385378
}
386379

0 commit comments

Comments
 (0)