Skip to content

Commit e5b8b45

Browse files
AbirAbbasclaude
andcommitted
feat(sdk/typescript): model#variant reasoning-effort parity in harness providers
Mirrors the Python SDK: splitModelVariant/resolveModelAndVariant helper, variant?: string through HarnessConfig/HarnessOptions, opencode passes --variant, codex now passes -m at all (it previously sent no model to the CLI — same bug the Python provider had) plus -c model_reasoning_effort, claude/gemini strip the suffix. Attribution overlay and usage metrics key off the base model. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 5620b4e commit e5b8b45

16 files changed

Lines changed: 389 additions & 12 deletions

sdk/typescript/src/agent/Agent.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ import { AIClient } from '../ai/AIClient.js';
4141
import { AgentFieldClient } from '../client/AgentFieldClient.js';
4242
import type { HarnessRunner } from '../harness/runner.js';
4343
import type { HarnessOptions, HarnessResult } from '../harness/types.js';
44+
import { splitModelVariant } from '../harness/modelVariant.js';
4445
import { MemoryClient } from '../memory/MemoryClient.js';
4546
import { MemoryEventClient } from '../memory/MemoryEventClient.js';
4647
import {
@@ -412,9 +413,13 @@ export class Agent {
412413

413414
const providerName = options?.provider ?? this.config.harnessConfig?.provider;
414415
const harnessName = providerName ? String(providerName).replace(/-/g, '_') : null;
415-
const modelName = String(
416+
// Usage is recorded against the base model — a "#variant"
417+
// reasoning-effort suffix on the configured model never reaches the
418+
// provider and must not reach the tracker either.
419+
const rawModelName = String(
416420
result.model ?? options?.model ?? this.config.harnessConfig?.model ?? providerName ?? 'harness'
417421
);
422+
const modelName = splitModelVariant(rawModelName).model ?? rawModelName;
418423

419424
tracker.record({
420425
model: modelName,
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
export type { HarnessConfig, HarnessOptions, Metrics, RawResult, HarnessResult } from './types.js';
22
export { createHarnessResult, createMetrics, createRawResult } from './types.js';
3+
export type { ModelVariant } from './modelVariant.js';
4+
export { MODEL_VARIANT_SEP, splitModelVariant, resolveModelAndVariant } from './modelVariant.js';
35
export type { HarnessProvider } from './providers/base.js';
46
export { buildProvider, SUPPORTED_PROVIDERS } from './providers/factory.js';
57
export { HarnessRunner } from './runner.js';
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/**
2+
* Separator for the `provider/model#variant` model-string syntax. `#` is
3+
* safe because provider model ids never contain it (`:` is taken by
4+
* OpenRouter suffixes like `:free`, `@` by Vertex-style ids).
5+
*/
6+
export const MODEL_VARIANT_SEP = '#';
7+
8+
export interface ModelVariant {
9+
model?: string;
10+
variant?: string;
11+
}
12+
13+
/**
14+
* Split a `provider/model#variant` string into `{ model, variant }`.
15+
*
16+
* The `#variant` suffix carries a provider-specific reasoning-effort
17+
* variant (e.g. `high`, `minimal`) through config surfaces that only
18+
* hold a model string. A bare model string passes through as
19+
* `{ model }`; non-string or empty input yields `{}`.
20+
*/
21+
export function splitModelVariant(model: unknown): ModelVariant {
22+
if (typeof model !== 'string' || !model.trim()) {
23+
return {};
24+
}
25+
const sepIndex = model.indexOf(MODEL_VARIANT_SEP);
26+
const base = sepIndex === -1 ? model : model.slice(0, sepIndex);
27+
const variant = sepIndex === -1 ? '' : model.slice(sepIndex + 1);
28+
return {
29+
model: base.trim() || undefined,
30+
variant: variant.trim() || undefined,
31+
};
32+
}
33+
34+
/**
35+
* Resolve `{ model, variant }` from harness options.
36+
*
37+
* An explicit `options.variant` wins over a `#variant` suffix on
38+
* `options.model`; the returned model never carries the suffix.
39+
*/
40+
export function resolveModelAndVariant(options: Record<string, unknown>): ModelVariant {
41+
const { model, variant } = splitModelVariant(options.model);
42+
const explicit = options.variant;
43+
if (typeof explicit === 'string' && explicit.trim()) {
44+
return { model, variant: explicit.trim() };
45+
}
46+
return { model, variant };
47+
}

sdk/typescript/src/harness/providers/claude.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type { HarnessProvider } from './base.js';
22
import type { RawResult } from '../types.js';
33
import { createMetrics, createRawResult } from '../types.js';
4+
import { resolveModelAndVariant } from '../modelVariant.js';
45

56
type QueryInput = {
67
prompt: string;
@@ -39,7 +40,10 @@ export class ClaudeCodeProvider implements HarnessProvider {
3940
}
4041

4142
const agentOptions: Record<string, unknown> = {};
42-
if (options.model !== undefined) agentOptions.model = options.model;
43+
// claude-agent-sdk has no reasoning-effort knob; strip any "#variant"
44+
// suffix so the SDK receives a valid model id, and drop the variant.
45+
const { model: modelValue } = resolveModelAndVariant(options);
46+
if (modelValue !== undefined) agentOptions.model = modelValue;
4347
if (options.cwd !== undefined) agentOptions.cwd = options.cwd;
4448
if (options.maxTurns !== undefined) agentOptions.max_turns = options.maxTurns;
4549
if (options.tools !== undefined) agentOptions.allowed_tools = options.tools;

sdk/typescript/src/harness/providers/codex.ts

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { HarnessProvider } from './base.js';
22
import type { RawResult } from '../types.js';
33
import { createRawResult, createMetrics } from '../types.js';
44
import { runCli, parseJsonl, extractFinalText } from '../cli.js';
5+
import { resolveModelAndVariant } from '../modelVariant.js';
56

67
export class CodexProvider implements HarnessProvider {
78
private readonly bin: string;
@@ -19,6 +20,19 @@ export class CodexProvider implements HarnessProvider {
1920
if (options.permissionMode === 'auto') {
2021
cmd.push('--full-auto');
2122
}
23+
24+
// Model via -m; reasoning effort has no dedicated flag — it's the
25+
// model_reasoning_effort config key. The effort comes from a "#variant"
26+
// suffix on the model (or an explicit options.variant), e.g.
27+
// "gpt-5.3-codex#high".
28+
const { model: modelValue, variant: variantValue } = resolveModelAndVariant(options);
29+
if (modelValue) {
30+
cmd.push('-m', modelValue);
31+
}
32+
if (variantValue) {
33+
cmd.push('-c', `model_reasoning_effort=${variantValue}`);
34+
}
35+
2236
cmd.push(prompt);
2337

2438
const startApi = Date.now();
@@ -50,7 +64,7 @@ export class CodexProvider implements HarnessProvider {
5064
return createRawResult({
5165
result: resultText,
5266
messages: events,
53-
metrics: createMetrics({ durationApiMs: Date.now() - startApi, numTurns, sessionId }),
67+
metrics: createMetrics({ durationApiMs: Date.now() - startApi, numTurns, sessionId, model: modelValue }),
5468
isError,
5569
errorMessage: isError ? stderr.trim() : undefined,
5670
});

sdk/typescript/src/harness/providers/gemini.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { HarnessProvider } from './base.js';
22
import type { RawResult } from '../types.js';
33
import { createRawResult, createMetrics } from '../types.js';
44
import { runCli } from '../cli.js';
5+
import { resolveModelAndVariant } from '../modelVariant.js';
56

67
export class GeminiProvider implements HarnessProvider {
78
private readonly bin: string;
@@ -19,8 +20,11 @@ export class GeminiProvider implements HarnessProvider {
1920
if (options.permissionMode === 'auto') {
2021
cmd.push('--sandbox');
2122
}
22-
if (options.model) {
23-
cmd.push('-m', String(options.model));
23+
// gemini has no reasoning-effort flag; strip any "#variant" suffix so
24+
// the CLI still receives a valid model id.
25+
const { model: modelValue } = resolveModelAndVariant(options);
26+
if (modelValue) {
27+
cmd.push('-m', modelValue);
2428
}
2529
cmd.push('-p', prompt);
2630

sdk/typescript/src/harness/providers/opencode.ts

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { HarnessProvider } from './base.js';
22
import type { RawResult } from '../types.js';
33
import { createRawResult, createMetrics } from '../types.js';
44
import { runCli } from '../cli.js';
5+
import { resolveModelAndVariant } from '../modelVariant.js';
56
import {
67
isOpenRouterRequest,
78
openRouterAttributionHeaders,
@@ -31,9 +32,16 @@ export class OpenCodeProvider implements HarnessProvider {
3132

3233
const env: Record<string, string> = { ...(options.env as Record<string, string>) };
3334

34-
// Pass model via -m flag on the run subcommand (not env var).
35-
if (options.model) {
36-
cmd.push('-m', String(options.model));
35+
// Pass model via -m flag on the run subcommand (not env var). A
36+
// "#variant" suffix on the model (or an explicit options.variant) maps
37+
// to --variant — opencode's provider-specific reasoning effort (e.g.
38+
// high, max, minimal).
39+
const { model: modelValue, variant: variantValue } = resolveModelAndVariant(options);
40+
if (modelValue) {
41+
cmd.push('-m', modelValue);
42+
}
43+
if (variantValue) {
44+
cmd.push('--variant', variantValue);
3745
}
3846

3947
// Handle system prompt - prepend to user prompt since OpenCode
@@ -46,14 +54,15 @@ export class OpenCodeProvider implements HarnessProvider {
4654
// Prompt is the positional `message` arg to `opencode run`.
4755
cmd.push(effectivePrompt);
4856

49-
const explicitModel = typeof options.model === 'string' ? options.model : undefined;
57+
// The attribution overlay keys off the base model — a "#variant" suffix
58+
// would otherwise leak into the config's model slug.
5059
if (
51-
explicitModel &&
52-
isOpenRouterRequest({ model: explicitModel }) &&
60+
modelValue &&
61+
isOpenRouterRequest({ model: modelValue }) &&
5362
!env.OPENCODE_CONFIG_CONTENT &&
5463
!process.env.OPENCODE_CONFIG_CONTENT
5564
) {
56-
const modelSlug = explicitModel.slice('openrouter/'.length);
65+
const modelSlug = modelValue.slice('openrouter/'.length);
5766
const headers = openRouterAttributionHeaders({ env: { ...process.env, ...env } });
5867
if (modelSlug && Object.keys(headers).length > 0) {
5968
env.OPENCODE_CONFIG_CONTENT = JSON.stringify({
@@ -82,6 +91,7 @@ export class OpenCodeProvider implements HarnessProvider {
8291
durationApiMs: Date.now() - startApi,
8392
numTurns: resultText ? 1 : 0,
8493
sessionId: '',
94+
model: modelValue,
8595
}),
8696
isError,
8797
errorMessage: isError ? stderr.trim() : undefined,

sdk/typescript/src/harness/runner.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ export class HarnessRunner {
9393
for (const key of [
9494
'provider',
9595
'model',
96+
'variant',
9697
'maxTurns',
9798
'maxBudgetUsd',
9899
'maxRetries',

sdk/typescript/src/harness/types.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
export interface HarnessConfig {
22
provider: 'claude-code' | 'codex' | 'gemini' | 'opencode';
33
model?: string;
4+
/**
5+
* Provider-specific reasoning-effort variant (e.g. `high`, `minimal`).
6+
* Wins over a `#variant` suffix on `model`.
7+
*/
8+
variant?: string;
49
maxTurns?: number;
510
maxBudgetUsd?: number;
611
maxRetries?: number;
@@ -20,6 +25,11 @@ export interface HarnessConfig {
2025
export interface HarnessOptions {
2126
provider?: string;
2227
model?: string;
28+
/**
29+
* Provider-specific reasoning-effort variant (e.g. `high`, `minimal`).
30+
* Wins over a `#variant` suffix on `model`.
31+
*/
32+
variant?: string;
2333
maxTurns?: number;
2434
maxBudgetUsd?: number;
2535
maxRetries?: number;
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import { describe, it, expect } from 'vitest';
2+
3+
import {
4+
MODEL_VARIANT_SEP,
5+
resolveModelAndVariant,
6+
splitModelVariant,
7+
} from '../src/harness/modelVariant.js';
8+
9+
describe('splitModelVariant', () => {
10+
it('parses a #variant suffix into model and variant', () => {
11+
expect(splitModelVariant('openrouter/z-ai/glm-5.2#high')).toEqual({
12+
model: 'openrouter/z-ai/glm-5.2',
13+
variant: 'high',
14+
});
15+
});
16+
17+
it('passes a bare model through with no variant', () => {
18+
expect(splitModelVariant('deepseek/deepseek-v4-flash')).toEqual({
19+
model: 'deepseek/deepseek-v4-flash',
20+
variant: undefined,
21+
});
22+
});
23+
24+
it('handles missing, empty, or non-string model', () => {
25+
expect(splitModelVariant(undefined)).toEqual({});
26+
expect(splitModelVariant(null)).toEqual({});
27+
expect(splitModelVariant('')).toEqual({});
28+
expect(splitModelVariant(' ')).toEqual({});
29+
expect(splitModelVariant(42)).toEqual({});
30+
});
31+
32+
it('treats empty halves around the separator as absent', () => {
33+
expect(splitModelVariant('model#')).toEqual({ model: 'model', variant: undefined });
34+
expect(splitModelVariant('#high')).toEqual({ model: undefined, variant: 'high' });
35+
});
36+
37+
it('splits on the first separator only', () => {
38+
expect(splitModelVariant('a#b#c')).toEqual({ model: 'a', variant: 'b#c' });
39+
});
40+
41+
it('trims whitespace around both halves', () => {
42+
expect(splitModelVariant(' openai/gpt-5 # high ')).toEqual({
43+
model: 'openai/gpt-5',
44+
variant: 'high',
45+
});
46+
});
47+
48+
it('uses # as the separator', () => {
49+
expect(MODEL_VARIANT_SEP).toBe('#');
50+
});
51+
});
52+
53+
describe('resolveModelAndVariant', () => {
54+
it('lets an explicit variant option win over the suffix', () => {
55+
expect(resolveModelAndVariant({ model: 'openai/gpt-5#low', variant: 'max' })).toEqual({
56+
model: 'openai/gpt-5',
57+
variant: 'max',
58+
});
59+
});
60+
61+
it('uses the suffix when no explicit option is set', () => {
62+
expect(resolveModelAndVariant({ model: 'openai/gpt-5#minimal' })).toEqual({
63+
model: 'openai/gpt-5',
64+
variant: 'minimal',
65+
});
66+
});
67+
68+
it('ignores a whitespace-only explicit variant', () => {
69+
expect(resolveModelAndVariant({ model: 'openai/gpt-5#low', variant: ' ' })).toEqual({
70+
model: 'openai/gpt-5',
71+
variant: 'low',
72+
});
73+
});
74+
75+
it('returns nothing for empty options', () => {
76+
expect(resolveModelAndVariant({})).toEqual({ model: undefined, variant: undefined });
77+
});
78+
});

0 commit comments

Comments
 (0)