From fa8bfa64d98f08e7ba32157606d22aa4fc618092 Mon Sep 17 00:00:00 2001 From: addyCooks Date: Tue, 25 Aug 2026 00:46:34 +0530 Subject: [PATCH 1/3] feat(estimate): add `sentinel estimate` and per-run model instrumentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers the first half of #1: how long an audit will take and roughly what it will cost, before it runs. `sentinel estimate` reports repositories, rule packs, files, model requests, tokens, and wall-clock runtime for a config, without invoking a model, filing anything, or mutating anything. The token figure is measured rather than guessed — the estimator assembles the same prompts the audit would send, via the same `buildAuditPrompt` and `applies_to` scoping, and counts them. What varies between installs is the per-request cost, so every run is now instrumented for calibration: each pack pass is timed, and its prompt and output tokens counted, into a `usage` block on the committed run record (requests, duration, prompt and output tokens). The estimator averages the last ten records to derive its per-request figures, and says so in the output when it is still on built-in defaults. Repos already checked out under the workspace are measured from their real files; `--clone` checks out the rest. A repo that contributed no files, a missing or unparseable pack, and a target that would not expand are each called out so a partial estimate never reads as the whole picture. Also extracts the target's pack resolution out of `runFromConfig` into `selectPacks`, so the estimate and the run agree on what would execute. No versioned contract is touched: the findings model, sentinel.yaml schema, and pack manifest are unchanged, and `usage` is optional on the run record so records written before this land still read. --- README.md | 2 +- docs/cli/index.md | 40 ++- docs/configuration/index.md | 2 + docs/index.md | 2 +- docs/workflow/index.md | 2 +- source/cli.ts | 99 +++++- source/index.ts | 20 +- source/observe/record.spec.ts | 42 ++- source/observe/record.ts | 17 + source/observe/types.ts | 17 + source/run/audit.spec.ts | 30 ++ source/run/audit.ts | 12 + source/run/estimate.spec.ts | 591 ++++++++++++++++++++++++++++++++++ source/run/estimate.ts | 339 +++++++++++++++++++ source/run/report.spec.ts | 1 + source/run/run.ts | 35 +- source/run/select.spec.ts | 40 ++- source/run/select.ts | 53 ++- source/run/types.ts | 16 + 19 files changed, 1302 insertions(+), 58 deletions(-) create mode 100644 source/run/estimate.spec.ts create mode 100644 source/run/estimate.ts diff --git a/README.md b/README.md index 101d661..ee6454f 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Full documentation is available online at **[docs.nanocollective.org](https://do - **[Configuration](docs/configuration/index.md)** — The `sentinel.yaml` reference - **[Workflow](docs/workflow/index.md)** — The scheduled audit, run modes, and the execution model - **[Findings & Issues](docs/findings/index.md)** — The severity model, issue filing, dedup, and suppression -- **[CLI](docs/cli/index.md)** — `init` and `run` command reference +- **[CLI](docs/cli/index.md)** — `init`, `run`, and `estimate` command reference - **[Community](docs/community.md)** — Contributing, Discord, and how to help ## What Sentinel is not (in v1) diff --git a/docs/cli/index.md b/docs/cli/index.md index 737871b..cc6accf 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -6,7 +6,7 @@ sidebar_order: 7 # CLI -The `@nanocollective/sentinel` package is both the scaffolder and the runtime. It exposes two commands: `init` scaffolds a config repo, and `run` performs an audit. The scheduled [workflow](../workflow/index.md) invokes `run` under the hood, and you can invoke either directly. +The `@nanocollective/sentinel` package is both the scaffolder and the runtime. It exposes three commands: `init` scaffolds a config repo, `run` performs an audit, and `estimate` sizes an audit before you run it. The scheduled [workflow](../workflow/index.md) invokes `run` under the hood, and you can invoke any of them directly. ```bash npx @nanocollective/sentinel [options] @@ -57,3 +57,41 @@ npx @nanocollective/sentinel run \ A local `run` **writes findings to a Markdown file and never files issues** — issue filing needs a GitHub token, which is only present in the Actions path. This makes local `run` the [calibration path](../rule-packs/authoring.md#calibrate-before-you-file) for pack authors: iterate on a pack against a real repo, read the Markdown, adjust, repeat, all without touching anyone's issue tracker. The same validator, dedup logic, and findings model apply in both contexts, so what you see locally is what the workflow will produce. + +## `estimate` + +Sizes an audit **before** it runs: how many repositories and rule packs are in scope, how many files they put in front of the model, and roughly how many model requests, tokens, and minutes that costs. It runs no model, files nothing, and mutates nothing — useful when you are about to point Sentinel at a dozen more repositories, or adding a pack to every target and want to know what that does to the nightly window. + +```bash +npx @nanocollective/sentinel estimate +``` + +```markdown +# Sentinel audit estimate + +- **Repositories:** 18 +- **Rule packs:** 5 +- **Files:** 1,204 +- **Estimated AI requests:** ~420 +- **Estimated tokens:** ~3.8M +- **Estimated runtime:** ~14 minute(s) + +Calibrated from the last 6 run record(s). +``` + +| Flag | Description | +| --- | --- | +| `--config ` | Path to `sentinel.yaml`. Defaults to `./sentinel.yaml`. | +| `--packs-dir ` | Rule packs directory. Defaults to `rule-packs/` beside the config. | +| `--workspace ` | Where the target repos are checked out. Defaults to `.`. | +| `--records-dir ` | Run records to calibrate from. Defaults to `runs`. | +| `--clone` | Check out any target repo not already present in the workspace. | +| `--output ` | Write the Markdown estimate here. Defaults to stdout. | + +### How the figures are produced + +The token figure is **measured, not guessed**: `estimate` assembles the same prompts the audit would send — the pack body, the reporting contract, and the source files scoped by each pack's `applies_to.paths` — and counts them. What varies between installs is the per-request cost, so the request, token, and runtime figures are calibrated from the run records the last ten runs committed. Every run is instrumented for this: it records how long each pack pass took, how many model requests it made (auto-fix retries included), and the tokens it sent and received. + +Until a run has been recorded, the figures fall back to built-in defaults, and the output says so. Treat a first, uncalibrated estimate as an order of magnitude rather than a number to schedule against. + +Repositories already checked out under `--workspace` are measured from their real files. Any that are not are counted with zero files and called out in the output, so a partial estimate never reads as the whole picture — pass `--clone` to check the rest out first. diff --git a/docs/configuration/index.md b/docs/configuration/index.md index f325e6c..0202c6d 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -86,6 +86,8 @@ Systematic noise on one repository is handled with an opt-in `sentinel.yaml` pla Cost scales with repositories × rule packs × schedule frequency. Twenty repos with four packs each, run daily, is eighty model calls a day. Local models keep that cost at zero; cloud models do not. Keep the default configuration modest, lean on local models for the routine passes, and reserve a cloud fallback for the cases that genuinely need it. +To put numbers on a specific config before committing to it, run [`sentinel estimate`](../cli/index.md#estimate) — it reports the requests, tokens, and runtime the config implies, calibrated from your own recorded runs. + ## Observability Each run commits a run record to the config repo and writes a step summary. A lightweight static dashboard is generated into the config repo's GitHub Pages from those committed records — no database. See [Workflow → observability](../workflow/index.md#observability-and-run-history). diff --git a/docs/index.md b/docs/index.md index 97ff232..7fe11b1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,7 +41,7 @@ npx @nanocollective/sentinel init - [Configuration](configuration/index.md) — The `sentinel.yaml` reference - [Workflow](workflow/index.md) — The scheduled audit, run modes, and the execution model - [Findings & Issues](findings/index.md) — The severity model, issue filing, dedup, and suppression -- [CLI](cli/index.md) — `init` and `run` command reference +- [CLI](cli/index.md) — `init`, `run`, and `estimate` command reference - [Community](community.md) — Get involved > Sentinel is in active development toward its v1. These docs describe the v1 design settled in the [Sentinel whitepaper](/collective/whitepapers/sentinel). Where a feature is planned rather than shipped, the docs say so. diff --git a/docs/workflow/index.md b/docs/workflow/index.md index 06c4475..000699f 100644 --- a/docs/workflow/index.md +++ b/docs/workflow/index.md @@ -64,7 +64,7 @@ Sentinel is not a model and does not train one. It uses whichever Nanocoder-conf There is no database. Run history is: - **Step summary** — the immediate run's result, in the Actions run. -- **Run record** — a committed record per run, in the config repo. This is the durable store. +- **Run record** — a committed record per run, in the config repo. This is the durable store. Alongside the findings it carries what the run cost the model: requests made, wall-clock time, and tokens sent and received. [`sentinel estimate`](../cli/index.md#estimate) reads those back to calibrate its figures. - **Dashboard** — a lightweight static site generated into the config repo's GitHub Pages from the committed run records. The read-side surface for trends: per-pack hit rates, findings over time, and aggregate model cost per run. ## Triggers in v1 diff --git a/source/cli.ts b/source/cli.ts index 525cfaa..d63a601 100644 --- a/source/cli.ts +++ b/source/cli.ts @@ -3,8 +3,9 @@ /** * Sentinel CLI entry point. * - * sentinel init scaffold a configuration repo - * sentinel run perform an audit pass (not yet implemented) + * sentinel init scaffold a configuration repo + * sentinel run perform an audit pass + * sentinel estimate size an audit before running it * * The init command's logic lives in ./init; this file is the interactive glue * (prompting, printing) and is excluded from coverage. @@ -30,6 +31,7 @@ import {buildRunRecord, recordFilename} from './observe/record.js'; import type {RunMode, RunRecord} from './observe/types.js'; import {nanocoderRunner} from './orchestrator/nanocoder-runner.js'; import {prepareRepo} from './run/clone.js'; +import {estimateRun, renderEstimate} from './run/estimate.js'; import {renderPreview} from './run/preview.js'; import {ghRepoLister} from './run/repo-lister.js'; import {renderReport} from './run/report.js'; @@ -39,8 +41,9 @@ import {fsPackLoader, fsRepoFiles} from './run/sources.js'; const USAGE = `sentinel Commands: - init Scaffold a Sentinel configuration into the current repository - run Perform an audit pass against a rule pack and a repository + init Scaffold a Sentinel configuration into the current repository + run Perform an audit pass against a rule pack and a repository + estimate Size an audit — requests, tokens, runtime — without running it Run 'sentinel --help' for command-specific options.`; @@ -186,20 +189,26 @@ function writeRunRecord(record: RunRecord, recordsDir: string): void { console.log(`Wrote run record to ${path}`); } -function writeDashboard(recordsDir: string, dashboardDir: string): void { +function readRunRecords(recordsDir: string): RunRecord[] { const records: RunRecord[] = []; - if (existsSync(recordsDir)) { - for (const name of readdirSync(recordsDir)) { - if (!name.endsWith('.json')) { - continue; - } - try { - records.push(JSON.parse(readFileSync(join(recordsDir, name), 'utf8'))); - } catch { - // Skip a malformed record rather than fail the whole dashboard. - } + if (!existsSync(recordsDir)) { + return records; + } + for (const name of readdirSync(recordsDir)) { + if (!name.endsWith('.json')) { + continue; + } + try { + records.push(JSON.parse(readFileSync(join(recordsDir, name), 'utf8'))); + } catch { + // Skip a malformed record rather than fail the whole read. } } + return records; +} + +function writeDashboard(recordsDir: string, dashboardDir: string): void { + const records = readRunRecords(recordsDir); mkdirSync(dashboardDir, {recursive: true}); const path = join(dashboardDir, 'index.html'); writeFileSync(path, renderDashboard(records)); @@ -326,6 +335,64 @@ async function runRun(argv: string[]): Promise { return 0; } +const ESTIMATE_USAGE = `sentinel estimate [options] + +Size an audit before running it: repositories, rule packs, files, model +requests, tokens, and wall-clock runtime. Runs no model, files nothing, and +mutates nothing. + +Figures are calibrated from the committed run records when any exist, so they +sharpen against your own hardware and model. Repos already checked out under +the workspace are measured from their real files; pass --clone to check out the +rest. + +Options: + --config Path to sentinel.yaml (default ./sentinel.yaml) + --packs-dir Rule packs directory (default ./rule-packs) + --workspace Where target repos are checked out (default .) + --records-dir Run records to calibrate from (default runs) + --clone Check out any target repo not already present + --output Write the Markdown estimate here (default stdout)`; + +async function runEstimate(argv: string[]): Promise { + if (argv.includes('--help') || argv.includes('-h')) { + console.log(ESTIMATE_USAGE); + return 0; + } + const flags = flagMap(argv); + + const configPath = flagStr(flags, 'config') ?? 'sentinel.yaml'; + const parsed = parseConfig(readFileSync(configPath, 'utf8')); + if (!parsed.valid || !parsed.config) { + for (const error of parsed.errors) { + console.error(`config error — ${error.field}: ${error.message}`); + } + return 1; + } + + const estimate = await estimateRun( + parsed.config, + { + files: fsRepoFiles, + packs: fsPackLoader, + repoLister: ghRepoLister, + cloneRepo: flags.get('clone') === true ? prepareRepo : undefined, + records: readRunRecords(flagStr(flags, 'records-dir') ?? 'runs'), + }, + { + workspaceDir: flagStr(flags, 'workspace') ?? '.', + packsDir: + flagStr(flags, 'packs-dir') ?? join(dirname(configPath), 'rule-packs'), + }, + ); + + writeReport(renderEstimate(estimate), flagStr(flags, 'output')); + for (const error of estimate.targetErrors) { + console.error(`target: ${error}`); + } + return 0; +} + async function main(argv: string[]): Promise { const [command, ...rest] = argv; @@ -334,6 +401,8 @@ async function main(argv: string[]): Promise { return runInit(rest); case 'run': return runRun(rest); + case 'estimate': + return runEstimate(rest); case undefined: case '--help': case '-h': diff --git a/source/index.ts b/source/index.ts index 8257539..b628617 100644 --- a/source/index.ts +++ b/source/index.ts @@ -103,6 +103,7 @@ export type { RepoRunRecord, RunMode, RunRecord, + RunUsage, SeverityCounts, } from './observe/types.js'; export {runAudit} from './orchestrator/audit.js'; @@ -148,6 +149,17 @@ export { type PrepareResult, prepareRepo, } from './run/clone.js'; +export { + type AuditEstimate, + type Calibration, + calibrate, + type EstimateDeps, + type EstimateOptions, + estimateRun, + estimateTokens, + type RepoEstimate, + renderEstimate, +} from './run/estimate.js'; export { type ExpandResult, expandTargets, @@ -175,13 +187,19 @@ export { runFromConfig, runLocal, } from './run/run.js'; -export {isEnabledPackPath, unionPatterns} from './run/select.js'; +export { + isEnabledPackPath, + type SelectedPacks, + selectPacks, + unionPatterns, +} from './run/select.js'; export {fsPackLoader, fsRepoFiles} from './run/sources.js'; export type { LoadedPacks, PackLoadError, PackLoader, PackOutcome, + PackUsage, RepoFiles, RepoOutcome, RunOutcome, diff --git a/source/observe/record.spec.ts b/source/observe/record.spec.ts index 6d96d85..6218e86 100644 --- a/source/observe/record.spec.ts +++ b/source/observe/record.spec.ts @@ -20,7 +20,10 @@ function finding(severity: Finding['severity']): Finding { }; } -function pack(findings: Finding[]): PackOutcome { +function pack( + findings: Finding[], + overrides: Partial = {}, +): PackOutcome { return { pack: 'p', version: '1.0.0', @@ -28,6 +31,8 @@ function pack(findings: Finding[]): PackOutcome { attempts: 1, ok: true, errors: [], + usage: {durationMs: 1000, promptTokens: 400, outputTokens: 50}, + ...overrides, }; } @@ -107,6 +112,41 @@ test('carries target errors', t => { t.deepEqual(record.targetErrors, ['boom']); }); +test('totals the measured model usage across every pack pass', t => { + const r = report({ + outcome: { + repos: [ + repo('org/a', [ + pack([], { + attempts: 2, + usage: {durationMs: 3000, promptTokens: 800, outputTokens: 90}, + }), + pack([]), + ]), + repo('org/b', [pack([])]), + ], + }, + }); + const record = buildRunRecord(r, TS, 'audit-only'); + // 2 attempts on the first pass, 1 on each of the others. + t.deepEqual(record.totals.usage, { + requests: 4, + durationMs: 5000, + promptTokens: 1600, + outputTokens: 190, + }); +}); + +test('records zeroed usage when nothing was audited', t => { + const record = buildRunRecord(report(), TS, 'dry-run'); + t.deepEqual(record.totals.usage, { + requests: 0, + durationMs: 0, + promptTokens: 0, + outputTokens: 0, + }); +}); + test('recordFilename is filesystem-safe', t => { t.is(recordFilename(TS), '2026-07-21T06-00-00-000Z.json'); t.false(recordFilename(TS).includes(':')); diff --git a/source/observe/record.ts b/source/observe/record.ts index d2b1bf5..72c19d8 100644 --- a/source/observe/record.ts +++ b/source/observe/record.ts @@ -11,6 +11,7 @@ import type { RepoRunRecord, RunMode, RunRecord, + RunUsage, SeverityCounts, } from './types.js'; @@ -57,6 +58,21 @@ export function buildRunRecord( totalFindings += repo.findings; } + const usage: RunUsage = { + requests: 0, + durationMs: 0, + promptTokens: 0, + outputTokens: 0, + }; + for (const repo of report.outcome.repos) { + for (const pack of repo.packs) { + usage.requests += pack.attempts; + usage.durationMs += pack.usage.durationMs; + usage.promptTokens += pack.usage.promptTokens; + usage.outputTokens += pack.usage.outputTokens; + } + } + const record: RunRecord = { timestamp, mode, @@ -65,6 +81,7 @@ export function buildRunRecord( repos: repos.length, findings: totalFindings, bySeverity: totalsBySeverity, + usage, }, targetErrors: report.targetErrors, }; diff --git a/source/observe/types.ts b/source/observe/types.ts index 7b02532..afb8f34 100644 --- a/source/observe/types.ts +++ b/source/observe/types.ts @@ -29,6 +29,21 @@ export interface RepoRunRecord { /** The mode a run executed in. */ export type RunMode = 'live' | 'dry-run' | 'audit-only'; +/** + * What a run cost the model, aggregated across every pack pass. `sentinel + * estimate` reads these back to calibrate its figures. + */ +export interface RunUsage { + /** Model invocations, auto-fix retries included. */ + requests: number; + /** Wall-clock milliseconds spent in the model. */ + durationMs: number; + /** Estimated prompt tokens sent. */ + promptTokens: number; + /** Estimated tokens returned. */ + outputTokens: number; +} + /** Issue-filing totals for a live run. */ export interface FilingSummary { filed: number; @@ -46,6 +61,8 @@ export interface RunRecord { repos: number; findings: number; bySeverity: SeverityCounts; + /** Absent on records written before runs were instrumented. */ + usage?: RunUsage; }; /** Present on a live run. */ filing?: FilingSummary; diff --git a/source/run/audit.spec.ts b/source/run/audit.spec.ts index 502d6cb..24b4376 100644 --- a/source/run/audit.spec.ts +++ b/source/run/audit.spec.ts @@ -58,6 +58,36 @@ test('produces a PackOutcome with findings from the model', async t => { t.true(r.prompt?.includes('src/a.ts')); }); +test('measures the pass so estimates have something to calibrate on', async t => { + const outcome = await auditPack( + PACK, + {repoName: 'org/a', files: [{path: 'src/a.ts', content: 'const x = 1;'}]}, + MODEL, + runner({ok: true, output: JSON.stringify([FINDING])}), + ); + t.true(outcome.usage.durationMs >= 0); + t.true(outcome.usage.promptTokens > 0); + t.true(outcome.usage.outputTokens > 0); +}); + +test('counts the prompt once per attempt', async t => { + // Malformed output the auto-fix loop retries, so two attempts are made. + const single = await auditPack( + PACK, + {repoName: 'org/a', files: []}, + MODEL, + runner({ok: true, output: JSON.stringify([FINDING])}), + ); + const retried = await auditPack( + PACK, + {repoName: 'org/a', files: []}, + MODEL, + runner({ok: true, output: 'not json'}), + ); + t.is(retried.attempts, 2); + t.is(retried.usage.promptTokens, single.usage.promptTokens * 2); +}); + test('surfaces a run error in the outcome', async t => { const outcome = await auditPack( PACK, diff --git a/source/run/audit.ts b/source/run/audit.ts index ad6fa41..ce724cf 100644 --- a/source/run/audit.ts +++ b/source/run/audit.ts @@ -1,6 +1,8 @@ /** * Run one rule pack against one repository's gathered files: build the prompt, * run the audit with the auto-fix loop, and map the result into a PackOutcome. + * The pass is timed and its tokens counted so `sentinel estimate` has real + * figures to calibrate against. */ import type {ModelConfig} from '../config/types.js'; @@ -12,6 +14,7 @@ import type {ModelRunner} from '../orchestrator/types.js'; import {buildAuditPrompt} from '../prompt/build.js'; import type {SourceFile} from '../prompt/types.js'; import type {RulePack} from '../rule-packs/types.js'; +import {estimateTokens} from './estimate.js'; import type {PackOutcome} from './types.js'; /** The repository material one pack pass audits. */ @@ -38,7 +41,9 @@ export async function auditPack( repoNotes: context.repoNotes, }); + const startedAt = Date.now(); const result = await runAuditWithAutoFix(prompt, model, runner, options); + const durationMs = Date.now() - startedAt; return { pack: pack.manifest.name, @@ -49,5 +54,12 @@ export async function auditPack( errors: result.errors, runError: result.runError, raw: result.raw, + usage: { + durationMs, + // A retry resends the audit prompt; the correction preamble is small + // beside it, so attempts x the base prompt is a fair figure. + promptTokens: estimateTokens(prompt) * result.attempts, + outputTokens: estimateTokens(result.raw), + }, }; } diff --git a/source/run/estimate.spec.ts b/source/run/estimate.spec.ts new file mode 100644 index 0000000..3fda711 --- /dev/null +++ b/source/run/estimate.spec.ts @@ -0,0 +1,591 @@ +import test from 'ava'; +import type {SentinelConfig} from '../config/types.js'; +import type {RunRecord, RunUsage} from '../observe/types.js'; +import type {SourceFile} from '../prompt/types.js'; +import type {RulePack} from '../rule-packs/types.js'; +import type {PrepareResult} from './clone.js'; +import { + type AuditEstimate, + calibrate, + estimateRun, + estimateTokens, + renderEstimate, +} from './estimate.js'; +import type {RepoLister} from './repo-lister.js'; +import type {LoadedPacks, PackLoader, RepoFiles} from './types.js'; + +console.log('\nrun/estimate.spec.ts'); + +const TS = '2026-07-21T06:00:00.000Z'; + +function config(overrides: Partial = {}): SentinelConfig { + return { + targets: [{repo: 'my-org/a', rulePacks: ['p']}], + schedule: '0 6 * * *', + severityThreshold: 'medium', + model: {provider: 'ollama', model: 'llama3.1'}, + issues: {label: 'sentinel', assignee: null, aggregateToConfigRepo: false}, + ...overrides, + }; +} + +function pack(name: string, dependsOn: string[] = []): RulePack { + return { + manifest: { + name, + version: '1.0.0', + description: '', + appliesTo: {paths: ['src/**/*.ts'], languages: ['typescript']}, + severityWeighting: {}, + dependsOn, + category: 'security', + }, + body: 'Flag bugs.', + }; +} + +function packLoader(loaded: LoadedPacks): PackLoader { + return { + async load(): Promise { + return loaded; + }, + }; +} + +function repoFiles(files: SourceFile[]): RepoFiles { + return { + async read(): Promise { + return files; + }, + async readText(): Promise { + return null; + }, + }; +} + +const FILES: SourceFile[] = [ + {path: 'src/a.ts', content: 'const x = 1;'}, + {path: 'src/b.ts', content: 'const y = 2;'}, +]; + +const OPTIONS = {workspaceDir: '/ws', packsDir: '/cfg/rule-packs'}; + +function record(usage: RunUsage | undefined, passes: number): RunRecord { + return { + timestamp: TS, + mode: 'live', + repos: [ + { + repo: 'my-org/a', + findings: 0, + bySeverity: {low: 0, medium: 0, high: 0, critical: 0}, + packs: Array.from({length: passes}, () => ({ + pack: 'p', + version: '1.0.0', + findings: 0, + ok: true, + })), + }, + ], + totals: { + repos: 1, + findings: 0, + bySeverity: {low: 0, medium: 0, high: 0, critical: 0}, + usage, + }, + targetErrors: [], + }; +} + +// --- estimateTokens --------------------------------------------------------- + +test('estimateTokens approximates four characters to a token', t => { + t.is(estimateTokens(''), 0); + t.is(estimateTokens('abcd'), 1); + // Rounds up: a partial token is still a token. + t.is(estimateTokens('abcde'), 2); +}); + +// --- calibrate -------------------------------------------------------------- + +test('calibrate falls back to built-in figures without records', t => { + const calibration = calibrate([]); + t.is(calibration.samples, 0); + t.true(calibration.msPerRequest > 0); + t.true(calibration.requestsPerPass >= 1); + t.true(calibration.outputTokensPerRequest > 0); +}); + +test('calibrate derives per-request figures from recorded usage', t => { + const calibration = calibrate([ + record( + {requests: 4, durationMs: 40_000, promptTokens: 8000, outputTokens: 800}, + 4, + ), + ]); + t.is(calibration.samples, 1); + t.is(calibration.msPerRequest, 10_000); + t.is(calibration.outputTokensPerRequest, 200); + t.is(calibration.requestsPerPass, 1); +}); + +test('calibrate reflects auto-fix retries in requests per pass', t => { + // Six requests across four pack passes: retries are part of the cost. + const calibration = calibrate([ + record( + {requests: 6, durationMs: 60_000, promptTokens: 9000, outputTokens: 600}, + 4, + ), + ]); + t.is(calibration.requestsPerPass, 1.5); +}); + +test('calibrate skips records with no usage or no requests', t => { + const calibration = calibrate([ + record(undefined, 2), + record({requests: 0, durationMs: 0, promptTokens: 0, outputTokens: 0}, 2), + ]); + t.is(calibration.samples, 0); +}); + +test('calibrate averages the most recent records', t => { + const older = record( + {requests: 1, durationMs: 30_000, promptTokens: 100, outputTokens: 100}, + 1, + ); + older.timestamp = '2026-07-20T06:00:00.000Z'; + const newer = record( + {requests: 1, durationMs: 10_000, promptTokens: 100, outputTokens: 300}, + 1, + ); + const calibration = calibrate([newer, older]); + t.is(calibration.samples, 2); + t.is(calibration.msPerRequest, 20_000); + t.is(calibration.outputTokensPerRequest, 200); +}); + +test('calibrate never reports fewer than one request per pass', t => { + // A record whose repos carry more packs than requests (a partial run). + const calibration = calibrate([ + record( + {requests: 1, durationMs: 5000, promptTokens: 100, outputTokens: 100}, + 4, + ), + ]); + t.is(calibration.requestsPerPass, 1); +}); + +test('calibrate keeps to the most recent window of records', t => { + const records = Array.from({length: 14}, (_, i) => { + const entry = record( + {requests: 1, durationMs: 1000, promptTokens: 100, outputTokens: 100}, + 1, + ); + entry.timestamp = `2026-07-${String(i + 1).padStart(2, '0')}T06:00:00.000Z`; + return entry; + }); + t.is(calibrate(records).samples, 10); +}); + +test('calibrate keeps the default retry rate when a record has no passes', t => { + const calibration = calibrate([ + record( + {requests: 2, durationMs: 4000, promptTokens: 200, outputTokens: 200}, + 0, + ), + ]); + t.is(calibration.msPerRequest, 2000); + t.is(calibration.requestsPerPass, calibrate([]).requestsPerPass); +}); + +// --- estimateRun ------------------------------------------------------------ + +test('estimates a target from the prompts an audit would send', async t => { + const estimate = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + }, + OPTIONS, + ); + t.is(estimate.totals.repos, 1); + t.is(estimate.totals.rulePacks, 1); + t.is(estimate.totals.files, 2); + t.true(estimate.totals.requests >= 1); + t.true(estimate.totals.tokens > 0); + t.true(estimate.totals.durationMs > 0); + t.deepEqual(estimate.repos[0]?.packs, ['p']); + t.is(estimate.repos[0]?.files, 2); +}); + +test('counts a pack dependency chain as extra passes', async t => { + const one = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + }, + OPTIONS, + ); + const two = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({ + packs: [pack('p', ['base']), pack('base')], + errors: [], + }), + }, + OPTIONS, + ); + t.is(two.totals.rulePacks, 2); + t.is(two.repos[0]?.packs.length, 2); + t.true(two.totals.requests > one.totals.requests); + t.true(two.totals.tokens > one.totals.tokens); +}); + +test('only counts files a pack actually sends', async t => { + const estimate = await estimateRun( + config(), + { + // README.md is outside the pack's applies_to scope. + files: repoFiles([...FILES, {path: 'README.md', content: '# hi'}]), + packs: packLoader({packs: [pack('p')], errors: []}), + }, + OPTIONS, + ); + t.is(estimate.totals.files, 2); +}); + +test('a bigger repository estimates more tokens', async t => { + const small = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + }, + OPTIONS, + ); + const large = await estimateRun( + config(), + { + files: repoFiles([{path: 'src/a.ts', content: 'x'.repeat(40_000)}]), + packs: packLoader({packs: [pack('p')], errors: []}), + }, + OPTIONS, + ); + t.true(large.totals.tokens > small.totals.tokens); +}); + +test('reports packs the rule-packs directory does not have', async t => { + const estimate = await estimateRun( + config({targets: [{repo: 'my-org/a', rulePacks: ['p', 'gone']}]}), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + }, + OPTIONS, + ); + t.deepEqual(estimate.repos[0]?.missingPacks, ['gone']); + t.is(estimate.repos[0]?.packs.length, 1); +}); + +test('carries pack load errors through to the estimate', async t => { + const estimate = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({ + packs: [pack('p')], + errors: [{file: 'broken.md', errors: []}], + }), + }, + OPTIONS, + ); + t.is(estimate.packLoadErrors.length, 1); +}); + +test('expands pattern targets through the repo lister', async t => { + const lister: RepoLister = { + async list(): Promise { + return ['my-org/web-one', 'my-org/web-two', 'my-org/api']; + }, + }; + const estimate = await estimateRun( + config({targets: [{pattern: 'my-org/web-*', rulePacks: ['p']}]}), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + repoLister: lister, + }, + OPTIONS, + ); + t.is(estimate.totals.repos, 2); +}); + +test('records a target that could not be expanded', async t => { + const estimate = await estimateRun( + config({targets: [{pattern: 'my-org/web-*', rulePacks: ['p']}]}), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + }, + OPTIONS, + ); + t.is(estimate.totals.repos, 0); + t.is(estimate.targetErrors.length, 1); +}); + +test('clones missing repos when a cloner is supplied', async t => { + const cloned: string[] = []; + const estimate = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + async cloneRepo(repo): Promise { + cloned.push(repo); + return {ok: true, skipped: false}; + }, + }, + OPTIONS, + ); + t.deepEqual(cloned, ['my-org/a']); + t.is(estimate.totals.repos, 1); +}); + +test('skips a repo that could not be checked out', async t => { + const estimate = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + async cloneRepo(): Promise { + return {ok: false, skipped: false, error: 'no such repo'}; + }, + }, + OPTIONS, + ); + t.is(estimate.totals.repos, 0); + t.true(estimate.targetErrors[0]?.includes('no such repo')); +}); + +test('uses recorded usage instead of the built-in defaults', async t => { + const deps = { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + }; + const uncalibrated = await estimateRun(config(), deps, OPTIONS); + const calibrated = await estimateRun( + config(), + { + ...deps, + records: [ + record( + { + requests: 1, + durationMs: 5000, + promptTokens: 100, + outputTokens: 100, + }, + 1, + ), + ], + }, + OPTIONS, + ); + t.is(calibrated.calibration.samples, 1); + t.is(calibrated.totals.durationMs, 5000); + t.not(calibrated.totals.durationMs, uncalibrated.totals.durationMs); +}); + +test('a repo audited by two targets is counted once', async t => { + const estimate = await estimateRun( + config({ + targets: [ + {repo: 'my-org/a', rulePacks: ['p']}, + {repo: 'my-org/a', rulePacks: ['q']}, + ], + }), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p'), pack('q')], errors: []}), + }, + OPTIONS, + ); + t.is(estimate.totals.repos, 1); + t.is(estimate.totals.rulePacks, 2); + t.is(estimate.repos[0]?.packs.length, 2); +}); + +// --- renderEstimate --------------------------------------------------------- + +async function estimateOf( + overrides: Partial = {}, + files: SourceFile[] = FILES, +): Promise { + return estimateRun( + config(overrides), + { + files: repoFiles(files), + packs: packLoader({packs: [pack('p')], errors: []}), + }, + OPTIONS, + ); +} + +test('renders the headline figures the issue asks for', async t => { + const markdown = renderEstimate(await estimateOf()); + t.true(markdown.startsWith('# Sentinel audit estimate')); + t.true(markdown.includes('**Repositories:** 1')); + t.true(markdown.includes('**Rule packs:** 1')); + t.true(markdown.includes('**Files:** 2')); + t.true(markdown.includes('**Estimated AI requests:**')); + t.true(markdown.includes('**Estimated tokens:**')); + t.true(markdown.includes('**Estimated runtime:**')); + t.true(markdown.includes('| `my-org/a` |')); +}); + +test('says when the figures are built-in rather than calibrated', async t => { + t.true(renderEstimate(await estimateOf()).includes('No run records yet')); +}); + +test('names the record count once calibrated', async t => { + const estimate = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + records: [ + record( + { + requests: 2, + durationMs: 20_000, + promptTokens: 400, + outputTokens: 200, + }, + 2, + ), + ], + }, + OPTIONS, + ); + t.true(renderEstimate(estimate).includes('Calibrated from the last 1 run')); +}); + +test('renders large figures compactly', t => { + const markdown = renderEstimate({ + repos: [ + { + repo: 'my-org/a', + packs: ['p'], + files: 1204, + requests: 420, + tokens: 3_800_000, + durationMs: 14 * 60_000, + missingPacks: [], + }, + ], + totals: { + repos: 18, + rulePacks: 5, + files: 1204, + requests: 420, + tokens: 3_800_000, + durationMs: 14 * 60_000, + }, + calibration: { + msPerRequest: 2000, + requestsPerPass: 1, + outputTokensPerRequest: 700, + samples: 3, + }, + packLoadErrors: [], + targetErrors: [], + }); + t.true(markdown.includes('**Files:** 1,204')); + t.true(markdown.includes('~420')); + t.true(markdown.includes('~3.8M')); + t.true(markdown.includes('~14 minute(s)')); +}); + +test('renders seconds, minutes, and hours as an operator reads them', t => { + const of = (durationMs: number, tokens: number): string => + renderEstimate({ + repos: [], + totals: { + repos: 0, + rulePacks: 0, + files: 0, + requests: 0, + tokens, + durationMs, + }, + calibration: { + msPerRequest: 0, + requestsPerPass: 1, + outputTokensPerRequest: 0, + samples: 1, + }, + packLoadErrors: [], + targetErrors: [], + }); + t.true(of(45_000, 800).includes('~45 second(s)')); + t.true(of(45_000, 800).includes('~800')); + t.true(of(20 * 60_000, 41_200).includes('~20 minute(s)')); + t.true(of(20 * 60_000, 41_200).includes('~41.2K')); + t.true(of(3 * 3_600_000, 0).includes('~3 hour(s)')); + t.true( + of(2 * 3_600_000 + 30 * 60_000, 0).includes('~2 hour(s) 30 minute(s)'), + ); +}); + +test('warns when a repo contributed no files', async t => { + const markdown = renderEstimate(await estimateOf({}, [])); + t.true(markdown.includes('contributed no files')); + t.true(markdown.includes('--clone')); +}); + +test('warns about missing packs, unparseable packs, and target errors', t => { + const markdown = renderEstimate({ + repos: [ + { + repo: 'my-org/a', + packs: ['p'], + files: 3, + requests: 1, + tokens: 100, + durationMs: 1000, + missingPacks: ['gone'], + }, + ], + totals: { + repos: 1, + rulePacks: 1, + files: 3, + requests: 1, + tokens: 100, + durationMs: 1000, + }, + calibration: { + msPerRequest: 1000, + requestsPerPass: 1, + outputTokensPerRequest: 100, + samples: 1, + }, + packLoadErrors: [{file: 'broken.md', errors: []}], + targetErrors: ['failed to list repos for "my-org"'], + }); + t.true(markdown.includes('rule pack(s) not in rule-packs/ — gone')); + t.true(markdown.includes('`broken.md` failed to parse')); + t.true(markdown.includes('failed to list repos')); +}); + +test('says so when no repositories resolved', async t => { + const estimate = await estimateOf({targets: []}); + t.is(estimate.totals.repos, 0); + const markdown = renderEstimate(estimate); + t.true(markdown.includes('No repositories resolved')); + t.false(markdown.includes('| Repository |')); +}); diff --git a/source/run/estimate.ts b/source/run/estimate.ts new file mode 100644 index 0000000..f73a9c7 --- /dev/null +++ b/source/run/estimate.ts @@ -0,0 +1,339 @@ +/** + * Pre-flight estimation for `sentinel estimate`. Answers the questions an + * operator asks before pointing Sentinel at an organisation — how many model + * requests and tokens will this cost, and how long will it take — without + * invoking a model (see docs/cli/index.md#estimate). + * + * The prompts are assembled exactly as an audit assembles them, so the token + * figure is measured rather than guessed. The per-request figures come from the + * committed run records when any exist, so an install's estimates sharpen + * against its own hardware and model instead of a built-in constant. + */ + +import {join} from 'node:path'; +import type {SentinelConfig} from '../config/types.js'; +import type {RunRecord, RunUsage} from '../observe/types.js'; +import {buildAuditPrompt} from '../prompt/build.js'; +import type {PrepareResult} from './clone.js'; +import {expandTargets} from './expand.js'; +import type {RepoLister} from './repo-lister.js'; +import {selectPacks, unionPatterns} from './select.js'; +import type {PackLoadError, PackLoader, RepoFiles} from './types.js'; + +/** Characters per token — a coarse average across code and English prose. */ +const CHARS_PER_TOKEN = 4; + +/** How many recent records to calibrate from; enough to smooth a slow run. */ +const CALIBRATION_WINDOW = 10; + +/** The per-request figures an estimate is built from. */ +export interface Calibration { + /** Wall-clock milliseconds one model request takes. */ + msPerRequest: number; + /** Requests per pack pass — above 1 when auto-fix retries are common. */ + requestsPerPass: number; + /** Tokens the model returns per request. */ + outputTokensPerRequest: number; + /** Run records the figures came from; 0 means the built-in defaults. */ + samples: number; +} + +/** + * Used until a run has been recorded: a minute-ish per request on a local + * model, and roughly one pass in ten needing the auto-fix retry. + */ +const DEFAULT_CALIBRATION: Calibration = { + msPerRequest: 45_000, + requestsPerPass: 1.1, + outputTokensPerRequest: 700, + samples: 0, +}; + +/** One repository's share of an estimate. */ +export interface RepoEstimate { + repo: string; + /** The packs that will run, `depends_on` chains included. */ + packs: string[]; + /** Distinct files at least one pack will send to the model. */ + files: number; + /** Model requests, retries included. */ + requests: number; + tokens: number; + durationMs: number; + /** Packs the target names that the rule-packs directory does not have. */ + missingPacks: string[]; +} + +/** Everything `sentinel estimate` computed. */ +export interface AuditEstimate { + repos: RepoEstimate[]; + totals: { + repos: number; + /** Distinct packs across every repository. */ + rulePacks: number; + files: number; + requests: number; + tokens: number; + durationMs: number; + }; + calibration: Calibration; + packLoadErrors: PackLoadError[]; + /** Target-expansion and clone failures. */ + targetErrors: string[]; +} + +/** Injected dependencies for an estimate. All reads, no model, no mutation. */ +export interface EstimateDeps { + files: RepoFiles; + packs: PackLoader; + /** Lists an owner's repos to expand pattern targets. */ + repoLister?: RepoLister; + /** Checks out missing target repos; omit to estimate from what is present. */ + cloneRepo?: (repo: string, dir: string) => Promise; + /** Prior run records, for calibration. */ + records?: RunRecord[]; +} + +/** Options for an estimate. Mirrors the run options it predicts. */ +export interface EstimateOptions { + /** Directory the target repos are checked out under. */ + workspaceDir: string; + /** The config repo's rule-packs directory. */ + packsDir: string; +} + +/** Approximate the token count of a piece of prompt or completion text. */ +export function estimateTokens(text: string): number { + return Math.ceil(text.length / CHARS_PER_TOKEN); +} + +/** + * Derive per-request figures from the most recent run records that carry usage. + * Records written before instrumentation, and runs that made no request, are + * skipped; with nothing usable left the built-in defaults stand. + */ +export function calibrate(records: RunRecord[]): Calibration { + const sorted = [...records].sort((a, b) => + b.timestamp.localeCompare(a.timestamp), + ); + + const usable: {usage: RunUsage; passes: number}[] = []; + for (const record of sorted) { + const usage = record.totals.usage; + if (!usage || usage.requests <= 0) { + continue; + } + usable.push({ + usage, + passes: record.repos.reduce( + (total, repo) => total + repo.packs.length, + 0, + ), + }); + if (usable.length === CALIBRATION_WINDOW) { + break; + } + } + + if (usable.length === 0) { + return {...DEFAULT_CALIBRATION}; + } + + let requests = 0; + let passes = 0; + let durationMs = 0; + let outputTokens = 0; + for (const sample of usable) { + requests += sample.usage.requests; + durationMs += sample.usage.durationMs; + outputTokens += sample.usage.outputTokens; + passes += sample.passes; + } + + return { + msPerRequest: durationMs / requests, + requestsPerPass: + passes > 0 + ? Math.max(1, requests / passes) + : DEFAULT_CALIBRATION.requestsPerPass, + outputTokensPerRequest: outputTokens / requests, + samples: usable.length, + }; +} + +/** Estimate what a config-driven run would cost, without running it. */ +export async function estimateRun( + config: SentinelConfig, + deps: EstimateDeps, + options: EstimateOptions, +): Promise { + const loaded = await deps.packs.load(options.packsDir); + const calibration = calibrate(deps.records ?? []); + + const expanded = await expandTargets(config.targets, deps.repoLister); + const targetErrors = [...expanded.errors]; + const repos: RepoEstimate[] = []; + + for (const target of expanded.targets) { + const repoDir = join(options.workspaceDir, target.repo); + + if (deps.cloneRepo) { + const prepared = await deps.cloneRepo(target.repo, repoDir); + if (!prepared.ok) { + targetErrors.push( + `could not check out ${target.repo}: ${prepared.error ?? 'clone failed'}`, + ); + continue; + } + } + + const {packs, missing} = selectPacks(loaded.packs, target.rulePacks); + const files = await deps.files.read(repoDir, unionPatterns(packs)); + + // Build the real prompts: what the model is sent is what we count. + const audited = new Set(); + let promptTokens = 0; + for (const pack of packs) { + const built = buildAuditPrompt({pack, files, repoName: target.repo}); + promptTokens += estimateTokens(built.prompt); + for (const path of built.includedFiles) { + audited.add(path); + } + } + + const requests = Math.round(packs.length * calibration.requestsPerPass); + repos.push({ + repo: target.repo, + packs: packs.map(pack => pack.manifest.name), + files: audited.size, + requests, + // A retry resends the prompt, so both sides scale with requests. + tokens: + Math.round(promptTokens * calibration.requestsPerPass) + + Math.round(requests * calibration.outputTokensPerRequest), + durationMs: Math.round(requests * calibration.msPerRequest), + missingPacks: missing, + }); + } + + const sum = (pick: (repo: RepoEstimate) => number): number => + repos.reduce((total, repo) => total + pick(repo), 0); + + return { + repos, + totals: { + repos: repos.length, + rulePacks: new Set(repos.flatMap(repo => repo.packs)).size, + files: sum(repo => repo.files), + requests: sum(repo => repo.requests), + tokens: sum(repo => repo.tokens), + durationMs: sum(repo => repo.durationMs), + }, + calibration, + packLoadErrors: loaded.errors, + targetErrors, + }; +} + +function formatInt(value: number): string { + return Math.round(value).toLocaleString('en-US'); +} + +/** Compact token counts: 812, 41.2K, 3.8M. */ +function formatTokens(value: number): string { + if (value >= 1_000_000) { + return `${(value / 1_000_000).toFixed(1)}M`; + } + if (value >= 1_000) { + return `${(value / 1_000).toFixed(1)}K`; + } + return String(Math.round(value)); +} + +/** Wall-clock, rounded to the unit an operator actually schedules in. */ +function formatDuration(ms: number): string { + const seconds = Math.round(ms / 1000); + if (seconds < 90) { + return `${seconds} second(s)`; + } + const minutes = Math.round(seconds / 60); + if (minutes < 90) { + return `${minutes} minute(s)`; + } + const hours = Math.floor(minutes / 60); + const rest = minutes % 60; + return rest === 0 ? `${hours} hour(s)` : `${hours} hour(s) ${rest} minute(s)`; +} + +function repoRow(repo: RepoEstimate): string { + return `| \`${repo.repo}\` | ${repo.packs.length} | ${formatInt(repo.files)} | ~${formatInt(repo.requests)} | ~${formatTokens(repo.tokens)} | ~${formatDuration(repo.durationMs)} |`; +} + +/** Notes that keep an understated estimate from reading as the whole picture. */ +function caveats(estimate: AuditEstimate): string[] { + const notes: string[] = []; + + const empty = estimate.repos.filter(repo => repo.files === 0); + if (empty.length > 0) { + notes.push( + `> ⚠️ ${empty.length} repo(s) contributed no files, so their cost is understated — check them out under the workspace, or pass --clone: ${empty.map(repo => repo.repo).join(', ')}`, + ); + } + for (const repo of estimate.repos) { + if (repo.missingPacks.length > 0) { + notes.push( + `> ⚠️ ${repo.repo}: rule pack(s) not in rule-packs/ — ${repo.missingPacks.join(', ')}`, + ); + } + } + for (const error of estimate.packLoadErrors) { + notes.push( + `> ⚠️ rule pack \`${error.file}\` failed to parse and will not run`, + ); + } + for (const error of estimate.targetErrors) { + notes.push(`> ⚠️ ${error}`); + } + return notes; +} + +/** Render an estimate as Markdown, for stdout or `--output`. */ +export function renderEstimate(estimate: AuditEstimate): string { + const {totals, calibration} = estimate; + + const parts = [ + '# Sentinel audit estimate', + [ + `- **Repositories:** ${formatInt(totals.repos)}`, + `- **Rule packs:** ${formatInt(totals.rulePacks)}`, + `- **Files:** ${formatInt(totals.files)}`, + `- **Estimated AI requests:** ~${formatInt(totals.requests)}`, + `- **Estimated tokens:** ~${formatTokens(totals.tokens)}`, + `- **Estimated runtime:** ~${formatDuration(totals.durationMs)}`, + ].join('\n'), + calibration.samples > 0 + ? `Calibrated from the last ${calibration.samples} run record(s).` + : 'No run records yet — these use built-in defaults and sharpen once runs are recorded.', + ]; + + if (estimate.repos.length === 0) { + parts.push('No repositories resolved from the config.'); + } else { + parts.push( + [ + '## Per repository', + '', + '| Repository | Packs | Files | Requests | Tokens | Runtime |', + '| --- | --- | --- | --- | --- | --- |', + ...estimate.repos.map(repoRow), + ].join('\n'), + ); + } + + const notes = caveats(estimate); + if (notes.length > 0) { + parts.push(notes.join('\n')); + } + + return parts.join('\n\n'); +} diff --git a/source/run/report.spec.ts b/source/run/report.spec.ts index 4293f18..c717918 100644 --- a/source/run/report.spec.ts +++ b/source/run/report.spec.ts @@ -29,6 +29,7 @@ function pack(overrides: Partial = {}): PackOutcome { attempts: 1, ok: true, errors: [], + usage: {durationMs: 0, promptTokens: 0, outputTokens: 0}, ...overrides, }; } diff --git a/source/run/run.ts b/source/run/run.ts index 2a5c0f9..20c851b 100644 --- a/source/run/run.ts +++ b/source/run/run.ts @@ -16,9 +16,7 @@ import {targetRepoFor} from '../issues/file.js'; import type {FilingContext, ReconcileClient} from '../issues/types.js'; import type {AutoFixOptions} from '../orchestrator/auto-fix.js'; import type {ModelRunner} from '../orchestrator/types.js'; -import {resolveDependencies} from '../rule-packs/dependencies.js'; import {parseRulePack} from '../rule-packs/parse.js'; -import type {RulePack} from '../rule-packs/types.js'; import {auditPack} from './audit.js'; import type {PrepareResult} from './clone.js'; import {expandTargets} from './expand.js'; @@ -28,7 +26,7 @@ import { previewReconciliation, } from './preview.js'; import type {RepoLister} from './repo-lister.js'; -import {unionPatterns} from './select.js'; +import {selectPacks, unionPatterns} from './select.js'; import type { PackLoadError, PackLoader, @@ -105,9 +103,6 @@ export async function runFromConfig( options: RunConfigOptions, ): Promise { const loaded = await deps.packs.load(options.packsDir); - const packByName = new Map( - loaded.packs.map(pack => [pack.manifest.name, pack]), - ); const repos: RepoOutcome[] = []; const reconciled: {repo: string; result: ReconcileResult}[] = []; @@ -133,30 +128,10 @@ export async function runFromConfig( } } - const resolvedNames = new Set(); - const missingPacks: string[] = []; - for (const name of target.rulePacks) { - if (!packByName.has(name)) { - missingPacks.push(name); - continue; - } - const resolved = resolveDependencies(loaded.packs, name); - if (resolved.errors.length > 0) { - missingPacks.push(name); - continue; - } - for (const resolvedName of resolved.order) { - resolvedNames.add(resolvedName); - } - } - - const resolvedPacks: RulePack[] = []; - for (const name of resolvedNames) { - const pack = packByName.get(name); - if (pack) { - resolvedPacks.push(pack); - } - } + const {packs: resolvedPacks, missing: missingPacks} = selectPacks( + loaded.packs, + target.rulePacks, + ); const files = await deps.files.read(repoDir, unionPatterns(resolvedPacks)); diff --git a/source/run/select.spec.ts b/source/run/select.spec.ts index 3aa23cd..44fe0fd 100644 --- a/source/run/select.spec.ts +++ b/source/run/select.spec.ts @@ -1,6 +1,6 @@ import test from 'ava'; import type {RulePack} from '../rule-packs/types.js'; -import {isEnabledPackPath, unionPatterns} from './select.js'; +import {isEnabledPackPath, selectPacks, unionPatterns} from './select.js'; console.log('\nrun/select.spec.ts'); @@ -19,21 +19,53 @@ test('non-markdown files are not packs', t => { t.false(isEnabledPackPath('config.yaml')); }); -function pack(paths: string[]): RulePack { +function pack(paths: string[], name = 'p', dependsOn: string[] = []): RulePack { return { manifest: { - name: 'p', + name, version: '1.0.0', description: '', appliesTo: {paths, languages: []}, severityWeighting: {}, - dependsOn: [], + dependsOn, category: '', }, body: 'audit', }; } +const NAMED = (name: string, dependsOn: string[] = []): RulePack => + pack(['a/**'], name, dependsOn); + +test('selectPacks resolves named packs and their dependencies', t => { + const {packs, missing} = selectPacks( + [NAMED('app', ['base']), NAMED('base')], + ['app'], + ); + t.deepEqual(packs.map(entry => entry.manifest.name).sort(), ['app', 'base']); + t.deepEqual(missing, []); +}); + +test('selectPacks de-duplicates a pack named twice', t => { + const {packs} = selectPacks( + [NAMED('app', ['base']), NAMED('base')], + ['app', 'base'], + ); + t.is(packs.length, 2); +}); + +test('selectPacks reports a name the directory does not have', t => { + const {packs, missing} = selectPacks([NAMED('app')], ['app', 'gone']); + t.deepEqual(missing, ['gone']); + t.is(packs.length, 1); +}); + +test('selectPacks reports a pack whose dependency chain does not resolve', t => { + const {packs, missing} = selectPacks([NAMED('app', ['absent'])], ['app']); + t.deepEqual(missing, ['app']); + t.is(packs.length, 0); +}); + test('unionPatterns collects every packs paths', t => { const patterns = unionPatterns([pack(['a/**']), pack(['b/**', 'a/**'])]); t.deepEqual(patterns.sort(), ['a/**', 'b/**']); diff --git a/source/run/select.ts b/source/run/select.ts index ba0d7fc..ab3694d 100644 --- a/source/run/select.ts +++ b/source/run/select.ts @@ -1,9 +1,12 @@ /** - * Pure helpers for selecting pack files and the file patterns to gather. A pack - * file is an enabled `.md` whose path contains no underscore-prefixed segment — - * the `_starter/` convention that keeps template packs from loading. + * Pure helpers for selecting the packs a run executes: which files in the + * rule-packs directory are packs at all, which packs a target resolves to, and + * which repository files they need gathered. A pack file is an enabled `.md` + * whose path contains no underscore-prefixed segment — the `_starter/` + * convention that keeps template packs from loading. */ +import {resolveDependencies} from '../rule-packs/dependencies.js'; import type {RulePack} from '../rule-packs/types.js'; /** True if a rule-packs-relative path is an enabled pack file. */ @@ -32,3 +35,47 @@ export function unionPatterns(packs: RulePack[]): string[] { } return [...patterns]; } + +/** The packs one target actually runs, and the names that did not resolve. */ +export interface SelectedPacks { + packs: RulePack[]; + /** Named packs missing from the directory or with an unresolvable chain. */ + missing: string[]; +} + +/** + * Resolve a target's rule pack names into the packs to run, pulling in each + * pack's `depends_on` chain and de-duplicating across names. + */ +export function selectPacks( + available: RulePack[], + names: string[], +): SelectedPacks { + const byName = new Map(available.map(pack => [pack.manifest.name, pack])); + const resolved = new Set(); + const missing: string[] = []; + + for (const name of names) { + if (!byName.has(name)) { + missing.push(name); + continue; + } + const chain = resolveDependencies(available, name); + if (chain.errors.length > 0) { + missing.push(name); + continue; + } + for (const resolvedName of chain.order) { + resolved.add(resolvedName); + } + } + + const packs: RulePack[] = []; + for (const name of resolved) { + const pack = byName.get(name); + if (pack) { + packs.push(pack); + } + } + return {packs, missing}; +} diff --git a/source/run/types.ts b/source/run/types.ts index e375c3d..15c364c 100644 --- a/source/run/types.ts +++ b/source/run/types.ts @@ -35,6 +35,20 @@ export interface PackLoader { load(packsDir: string): Promise; } +/** + * What one pack pass actually cost. Recorded on every run so `sentinel + * estimate` can calibrate its figures against this install's own hardware and + * model rather than a built-in guess. + */ +export interface PackUsage { + /** Wall-clock milliseconds the pass took, retries included. */ + durationMs: number; + /** Estimated prompt tokens sent across every attempt. */ + promptTokens: number; + /** Estimated tokens the model returned. */ + outputTokens: number; +} + /** The outcome of one pack's audit pass against one repository. */ export interface PackOutcome { pack: string; @@ -46,6 +60,8 @@ export interface PackOutcome { runError?: string; /** The raw model output, kept for diagnosing a failed audit. */ raw?: string; + /** What the pass cost, measured as it ran. */ + usage: PackUsage; } /** All pack outcomes for one repository. */ From 0d5a45f521fa606424b53069cf4fa32d3eb22591 Mon Sep 17 00:00:00 2001 From: addyCooks Date: Mon, 31 Aug 2026 11:58:45 +0530 Subject: [PATCH 2/3] fix(estimate): count every attempt, and price runtime by prompt size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups. Both blocking items understated cost in the same direction, and both are fixed from data the run records already carry. Output tokens counted only the final attempt while prompt tokens were multiplied by the attempt count, so a retried pass recorded prompt x2 and output x1. Because calibrate divides output by a request count that includes retries, every retry dragged outputTokensPerRequest down and estimate then understated output for everyone. runAuditWithAutoFix now reports promptChars and outputChars accumulated across attempts, so both sides are exact rather than the final attempt scaled up: the retry's prompt carries a correction section the x2 approximation missed, and its discarded first response cost tokens to generate. Characters rather than tokens keeps the approximation in one place, and keeps whole prompts from being retained on the result. Runtime was requests x a flat msPerRequest, which is prompt-size-blind — the exact case the command is pitched at. Calibration now carries a fixed msPerRequest plus a marginal msPerPromptToken, least-squares fitted across records when they differ in prompt size. When they cannot separate the terms — one record, or every run the same size — the measured average is split using the proportion the defaults imply, so the magnitude stays measured even where the shape is assumed. A negative fitted term falls back the same way. The defaults still sum to the previous 45s at a 2,500-token prompt. Also: - The --clone warning selected on the post-applies_to count, so a repo that was checked out but matched no files was told to clone what it already had. RepoEstimate carries the pre-scoping count and the two cases now get separate warnings; the second is the more useful, since it means a pack is pointed at a repo it cannot see. - Docs and --help claimed the command "mutates nothing" while documenting --clone. Reworded, and the runtime model is documented. - formatDuration handed over to minutes at 90s while rounding minutes from 60s, so "1 minute(s)" was unreachable: 89s rendered as seconds and 90s jumped to 2 minutes. - runEstimate printed targetErrors to stderr even when the report went to stdout, where the caveats already list them. Now only when --output redirects the report to a file. The 0 exit is left as it was, with a comment marking it deliberate rather than an oversight. Left alone: the unguarded readFileSync(configPath). It matches runRun, so fixing one without the other would just make the two inconsistent. Breaking change, from the original commit rather than this one: PackOutcome.usage is required and PackOutcome is exported from source/index.ts, so an external consumer constructing one breaks at compile time. --- docs/cli/index.md | 8 +- source/cli.ts | 17 ++- source/orchestrator/auto-fix.spec.ts | 41 +++++++ source/orchestrator/auto-fix.ts | 15 ++- source/run/audit.spec.ts | 37 ++++-- source/run/audit.ts | 12 +- source/run/estimate.spec.ts | 161 ++++++++++++++++++++++++++- source/run/estimate.ts | 154 +++++++++++++++++++++++-- 8 files changed, 406 insertions(+), 39 deletions(-) diff --git a/docs/cli/index.md b/docs/cli/index.md index cc6accf..d18525e 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -60,7 +60,7 @@ The same validator, dedup logic, and findings model apply in both contexts, so w ## `estimate` -Sizes an audit **before** it runs: how many repositories and rule packs are in scope, how many files they put in front of the model, and roughly how many model requests, tokens, and minutes that costs. It runs no model, files nothing, and mutates nothing — useful when you are about to point Sentinel at a dozen more repositories, or adding a pack to every target and want to know what that does to the nightly window. +Sizes an audit **before** it runs: how many repositories and rule packs are in scope, how many files they put in front of the model, and roughly how many model requests, tokens, and minutes that costs. It runs no model and files no issues — `--clone` is the one flag that writes anything, checking out missing repos. Useful when you are about to point Sentinel at a dozen more repositories, or adding a pack to every target and want to know what that does to the nightly window. ```bash npx @nanocollective/sentinel estimate @@ -90,8 +90,10 @@ Calibrated from the last 6 run record(s). ### How the figures are produced -The token figure is **measured, not guessed**: `estimate` assembles the same prompts the audit would send — the pack body, the reporting contract, and the source files scoped by each pack's `applies_to.paths` — and counts them. What varies between installs is the per-request cost, so the request, token, and runtime figures are calibrated from the run records the last ten runs committed. Every run is instrumented for this: it records how long each pack pass took, how many model requests it made (auto-fix retries included), and the tokens it sent and received. +The token figure is **measured, not guessed**: `estimate` assembles the same prompts the audit would send — the pack body, the reporting contract, and the source files scoped by each pack's `applies_to.paths` — and counts them. What varies between installs is the per-request cost, so the request, token, and runtime figures are calibrated from the run records the last ten runs committed. Every run is instrumented for this: it records how long each pack pass took, how many model requests it made (auto-fix retries included), and the tokens it sent and received across every attempt — an auto-fix retry resends the prompt and generates a second response, and both are counted. + +Runtime is **not** a flat per-request average. A request costs a fixed amount regardless of size plus an amount that tracks prompt size, and both terms are fitted from the records, so sizing a config far larger than anything you have run is not priced as though the prompts stayed the same. When the records cannot separate the two — a single run, or every run the same size — the measured average is split using the proportion the built-in defaults imply. Until a run has been recorded, the figures fall back to built-in defaults, and the output says so. Treat a first, uncalibrated estimate as an order of magnitude rather than a number to schedule against. -Repositories already checked out under `--workspace` are measured from their real files. Any that are not are counted with zero files and called out in the output, so a partial estimate never reads as the whole picture — pass `--clone` to check the rest out first. +Repositories already checked out under `--workspace` are measured from their real files. Any that are not are counted with zero files and called out in the output, so a partial estimate never reads as the whole picture — pass `--clone` to check the rest out first. A repo that *is* checked out but whose files no pack matches gets a separate warning: nothing needs cloning, but a pack is pointed at a repository it cannot see. diff --git a/source/cli.ts b/source/cli.ts index d63a601..19e46cd 100644 --- a/source/cli.ts +++ b/source/cli.ts @@ -338,8 +338,8 @@ async function runRun(argv: string[]): Promise { const ESTIMATE_USAGE = `sentinel estimate [options] Size an audit before running it: repositories, rule packs, files, model -requests, tokens, and wall-clock runtime. Runs no model, files nothing, and -mutates nothing. +requests, tokens, and wall-clock runtime. Runs no model and files no issues; +--clone is the one flag that writes anything, checking out missing repos. Figures are calibrated from the committed run records when any exist, so they sharpen against your own hardware and model. Repos already checked out under @@ -386,10 +386,17 @@ async function runEstimate(argv: string[]): Promise { }, ); - writeReport(renderEstimate(estimate), flagStr(flags, 'output')); - for (const error of estimate.targetErrors) { - console.error(`target: ${error}`); + const output = flagStr(flags, 'output'); + writeReport(renderEstimate(estimate), output); + // The estimate already renders these as caveats, so repeat them on stderr + // only when the report went to a file and nobody would otherwise see them. + if (output) { + for (const error of estimate.targetErrors) { + console.error(`target: ${error}`); + } } + // Deliberately 0 even when targets failed: estimate is advisory, and a + // partial estimate is still useful — the caveats say what is missing. return 0; } diff --git a/source/orchestrator/auto-fix.spec.ts b/source/orchestrator/auto-fix.spec.ts index 45aa2a9..0c1a209 100644 --- a/source/orchestrator/auto-fix.spec.ts +++ b/source/orchestrator/auto-fix.spec.ts @@ -129,3 +129,44 @@ test('buildAutoFixPrompt renders document-level errors without an index', t => { t.true(prompt.includes('- document: no array found')); t.false(prompt.includes('finding[-1]')); }); + +// The audit layer turns these into the prompt/output token figures `estimate` +// calibrates from, so they have to cover every attempt rather than the last. +test('reports the characters sent and received on a single attempt', async t => { + const output = JSON.stringify([GOOD]); + const runner = queuedRunner([{ok: true, output}]); + const result = await runAuditWithAutoFix('prompt', MODEL, runner); + t.is(result.promptChars, 'prompt'.length); + t.is(result.outputChars, output.length); +}); + +test('accumulates characters across a retry', async t => { + const first = JSON.stringify([BAD]); + const second = JSON.stringify([GOOD]); + const runner = queuedRunner([ + {ok: true, output: first}, + {ok: true, output: second}, + ]); + const result = await runAuditWithAutoFix('prompt', MODEL, runner); + + t.is(result.attempts, 2); + // Both responses counted, not just the one that validated. + t.is(result.outputChars, first.length + second.length); + // Both prompts counted, and the retry carried the correction section, so + // the total exceeds twice the original rather than matching it. + t.is( + result.promptChars, + (runner.prompts[0]?.length ?? 0) + (runner.prompts[1]?.length ?? 0), + ); + t.true(result.promptChars > 'prompt'.length * 2); +}); + +test('a process failure is not retried and counts one exchange', async t => { + const runner = queuedRunner([ + {ok: false, output: '', error: 'nanocoder missing'}, + ]); + const result = await runAuditWithAutoFix('prompt', MODEL, runner); + t.is(result.attempts, 1); + t.is(result.promptChars, 'prompt'.length); + t.is(result.outputChars, 0); +}); diff --git a/source/orchestrator/auto-fix.ts b/source/orchestrator/auto-fix.ts index 64b9239..d8c2b22 100644 --- a/source/orchestrator/auto-fix.ts +++ b/source/orchestrator/auto-fix.ts @@ -22,6 +22,15 @@ export interface AutoFixOptions extends RunnerOptions { export interface AutoFixResult extends AuditResult { /** Number of model runs performed (1 = succeeded or failed on first try). */ attempts: number; + /** + * Characters sent across every attempt. A retry resends the audit prompt plus + * a correction section, so this is the real total rather than a multiple of + * the first prompt. Carried as characters, not tokens, to keep the token + * approximation in one place ({@link ../run/estimate.js estimateTokens}). + */ + promptChars: number; + /** Characters returned across every attempt, the discarded ones included. */ + outputChars: number; } function formatErrors(errors: ValidationError[]): string { @@ -83,12 +92,16 @@ export async function runAuditWithAutoFix( let result = await runAudit(prompt, model, runner, options); let attempts = 1; + let promptChars = prompt.length; + let outputChars = result.raw.length; while (!result.ok && !result.runError && attempts < maxAttempts) { const fixPrompt = buildAutoFixPrompt(prompt, result); result = await runAudit(fixPrompt, model, runner, options); attempts++; + promptChars += fixPrompt.length; + outputChars += result.raw.length; } - return {...result, attempts}; + return {...result, attempts, promptChars, outputChars}; } diff --git a/source/run/audit.spec.ts b/source/run/audit.spec.ts index 24b4376..5a4ea10 100644 --- a/source/run/audit.spec.ts +++ b/source/run/audit.spec.ts @@ -3,6 +3,7 @@ import type {ModelConfig} from '../config/types.js'; import type {ModelRunner, ModelRunResult} from '../orchestrator/types.js'; import type {RulePack} from '../rule-packs/types.js'; import {auditPack} from './audit.js'; +import {estimateTokens} from './estimate.js'; console.log('\nrun/audit.spec.ts'); @@ -70,22 +71,44 @@ test('measures the pass so estimates have something to calibrate on', async t => t.true(outcome.usage.outputTokens > 0); }); -test('counts the prompt once per attempt', async t => { - // Malformed output the auto-fix loop retries, so two attempts are made. - const single = await auditPack( +test('counts every attempt on both the prompt and the output side', async t => { + // The same malformed output, run once and then with the retry allowed, so + // the only difference between the two is the second attempt. + const context = {repoName: 'org/a', files: []}; + const once = await auditPack( PACK, - {repoName: 'org/a', files: []}, + context, MODEL, - runner({ok: true, output: JSON.stringify([FINDING])}), + runner({ok: true, output: 'not json'}), + {maxAttempts: 1}, ); const retried = await auditPack( PACK, - {repoName: 'org/a', files: []}, + context, MODEL, runner({ok: true, output: 'not json'}), ); + + t.is(once.attempts, 1); t.is(retried.attempts, 2); - t.is(retried.usage.promptTokens, single.usage.promptTokens * 2); + // The discarded first response still cost tokens to generate. Counting only + // the final attempt understated output, and since calibration divides by a + // request count that includes retries, it dragged every estimate down. + t.is(retried.usage.outputTokens, once.usage.outputTokens * 2); + // The retry resends the prompt plus a correction section, so the prompt side + // is more than twice the first attempt rather than exactly twice. + t.true(retried.usage.promptTokens > once.usage.promptTokens * 2); +}); + +test('a pass that never retries counts one prompt and one response', async t => { + const outcome = await auditPack( + PACK, + {repoName: 'org/a', files: []}, + MODEL, + runner({ok: true, output: JSON.stringify([FINDING])}), + ); + t.is(outcome.attempts, 1); + t.is(outcome.usage.outputTokens, estimateTokens(JSON.stringify([FINDING]))); }); test('surfaces a run error in the outcome', async t => { diff --git a/source/run/audit.ts b/source/run/audit.ts index ce724cf..d540b80 100644 --- a/source/run/audit.ts +++ b/source/run/audit.ts @@ -14,7 +14,7 @@ import type {ModelRunner} from '../orchestrator/types.js'; import {buildAuditPrompt} from '../prompt/build.js'; import type {SourceFile} from '../prompt/types.js'; import type {RulePack} from '../rule-packs/types.js'; -import {estimateTokens} from './estimate.js'; +import {tokensFromChars} from './estimate.js'; import type {PackOutcome} from './types.js'; /** The repository material one pack pass audits. */ @@ -56,10 +56,12 @@ export async function auditPack( raw: result.raw, usage: { durationMs, - // A retry resends the audit prompt; the correction preamble is small - // beside it, so attempts x the base prompt is a fair figure. - promptTokens: estimateTokens(prompt) * result.attempts, - outputTokens: estimateTokens(result.raw), + // Both sides are the real totals across every attempt, not the final + // one scaled up: a retry resends the prompt plus a correction section, + // and the output it discards still cost tokens to generate. Counting + // them keeps calibration honest, since requests includes retries too. + promptTokens: tokensFromChars(result.promptChars), + outputTokens: tokensFromChars(result.outputChars), }, }; } diff --git a/source/run/estimate.spec.ts b/source/run/estimate.spec.ts index 3fda711..447d9f1 100644 --- a/source/run/estimate.spec.ts +++ b/source/run/estimate.spec.ts @@ -6,6 +6,7 @@ import type {RulePack} from '../rule-packs/types.js'; import type {PrepareResult} from './clone.js'; import { type AuditEstimate, + type Calibration, calibrate, estimateRun, estimateTokens, @@ -116,6 +117,11 @@ test('calibrate falls back to built-in figures without records', t => { t.true(calibration.outputTokensPerRequest > 0); }); +/** What one request costs at a given prompt size under a calibration. */ +function msAt(calibration: Calibration, promptTokens: number): number { + return calibration.msPerRequest + calibration.msPerPromptToken * promptTokens; +} + test('calibrate derives per-request figures from recorded usage', t => { const calibration = calibrate([ record( @@ -124,9 +130,52 @@ test('calibrate derives per-request figures from recorded usage', t => { ), ]); t.is(calibration.samples, 1); - t.is(calibration.msPerRequest, 10_000); t.is(calibration.outputTokensPerRequest, 200); t.is(calibration.requestsPerPass, 1); + // One record cannot separate fixed from per-token cost, so the split is + // assumed — but it must still reproduce the 10s/request actually measured + // at the 2,000 prompt tokens/request that record carried. + t.is(msAt(calibration, 2000), 10_000); + t.true(calibration.msPerRequest > 0); + t.true(calibration.msPerPromptToken > 0); +}); + +test('calibrate separates fixed from per-token cost across differing sizes', t => { + // 20s of fixed cost plus 10ms per prompt token, measured at two sizes. + const small = record( + {requests: 1, durationMs: 30_000, promptTokens: 1000, outputTokens: 100}, + 1, + ); + small.timestamp = '2026-07-20T06:00:00.000Z'; + const large = record( + {requests: 1, durationMs: 120_000, promptTokens: 10_000, outputTokens: 100}, + 1, + ); + const calibration = calibrate([large, small]); + t.is(calibration.samples, 2); + t.is(Math.round(calibration.msPerRequest), 20_000); + t.is(Math.round(calibration.msPerPromptToken), 10); + // And it reproduces both observations it was fitted from. + t.is(Math.round(msAt(calibration, 1000)), 30_000); + t.is(Math.round(msAt(calibration, 10_000)), 120_000); +}); + +test('calibrate falls back to a split when the fit would go negative', t => { + // A bigger prompt that ran faster — noise, not a real negative rate. + const fast = record( + {requests: 1, durationMs: 5000, promptTokens: 10_000, outputTokens: 100}, + 1, + ); + fast.timestamp = '2026-07-20T06:00:00.000Z'; + const slow = record( + {requests: 1, durationMs: 60_000, promptTokens: 1000, outputTokens: 100}, + 1, + ); + const calibration = calibrate([slow, fast]); + t.true(calibration.msPerRequest >= 0); + t.true(calibration.msPerPromptToken >= 0); + // Pooled average preserved: 32.5s per request at 5,500 prompt tokens. + t.is(Math.round(msAt(calibration, 5500)), 32_500); }); test('calibrate reflects auto-fix retries in requests per pass', t => { @@ -160,8 +209,10 @@ test('calibrate averages the most recent records', t => { ); const calibration = calibrate([newer, older]); t.is(calibration.samples, 2); - t.is(calibration.msPerRequest, 20_000); t.is(calibration.outputTokensPerRequest, 200); + // Both records are the same prompt size, so the terms cannot be separated + // and the pooled 20s/request average is split instead. + t.is(msAt(calibration, 100), 20_000); }); test('calibrate never reports fewer than one request per pass', t => { @@ -194,7 +245,7 @@ test('calibrate keeps the default retry rate when a record has no passes', t => 0, ), ]); - t.is(calibration.msPerRequest, 2000); + t.is(msAt(calibration, 100), 2000); t.is(calibration.requestsPerPass, calibrate([]).requestsPerPass); }); @@ -371,6 +422,53 @@ test('skips a repo that could not be checked out', async t => { t.true(estimate.targetErrors[0]?.includes('no such repo')); }); +test('runtime scales with prompt size, not just request count', async t => { + // Calibrated on a small config, then asked to size a much larger one. The + // request count is identical, so a flat per-request average would report the + // same runtime for both — the failure this term exists to prevent. + const records = [ + record( + {requests: 1, durationMs: 30_000, promptTokens: 1000, outputTokens: 100}, + 1, + ), + ]; + const line = 'const x: number = 1;\n'; + const small = await estimateRun( + config(), + { + files: repoFiles([{path: 'src/a.ts', content: line}]), + packs: packLoader({packs: [pack('p')], errors: []}), + records, + }, + OPTIONS, + ); + const large = await estimateRun( + config(), + { + files: repoFiles([{path: 'src/a.ts', content: line.repeat(5000)}]), + packs: packLoader({packs: [pack('p')], errors: []}), + records, + }, + OPTIONS, + ); + + t.is(small.totals.requests, large.totals.requests); + t.true(large.totals.tokens > small.totals.tokens * 10); + t.true(large.totals.durationMs > small.totals.durationMs * 5); +}); + +test('an uncalibrated estimate still scales with prompt size', async t => { + // The built-in defaults carry the per-token term too, so the first estimate + // an install ever runs is not prompt-size-blind either. + const line = 'const x: number = 1;\n'; + const small = await estimateOf({}, [{path: 'src/a.ts', content: line}]); + const large = await estimateOf({}, [ + {path: 'src/a.ts', content: line.repeat(5000)}, + ]); + t.is(small.calibration.samples, 0); + t.true(large.totals.durationMs > small.totals.durationMs * 5); +}); + test('uses recorded usage instead of the built-in defaults', async t => { const deps = { files: repoFiles(FILES), @@ -396,8 +494,8 @@ test('uses recorded usage instead of the built-in defaults', async t => { OPTIONS, ); t.is(calibrated.calibration.samples, 1); - t.is(calibrated.totals.durationMs, 5000); t.not(calibrated.totals.durationMs, uncalibrated.totals.durationMs); + t.true(calibrated.totals.durationMs > 0); }); test('a repo audited by two targets is counted once', async t => { @@ -524,6 +622,7 @@ test('renders seconds, minutes, and hours as an operator reads them', t => { }, calibration: { msPerRequest: 0, + msPerPromptToken: 0, requestsPerPass: 1, outputTokensPerRequest: 0, samples: 1, @@ -535,18 +634,36 @@ test('renders seconds, minutes, and hours as an operator reads them', t => { t.true(of(45_000, 800).includes('~800')); t.true(of(20 * 60_000, 41_200).includes('~20 minute(s)')); t.true(of(20 * 60_000, 41_200).includes('~41.2K')); + // Handover at 60s, not 90s: rounding minutes from 89s gave "1 minute(s)" + // no window at all, so the output jumped 89 second(s) -> 2 minute(s). + t.true(of(59_000, 0).includes('~59 second(s)')); + t.true(of(60_000, 0).includes('~1 minute(s)')); + t.true(of(89_000, 0).includes('~1 minute(s)')); t.true(of(3 * 3_600_000, 0).includes('~3 hour(s)')); t.true( of(2 * 3_600_000 + 30 * 60_000, 0).includes('~2 hour(s) 30 minute(s)'), ); }); -test('warns when a repo contributed no files', async t => { +test('warns when a repo is not checked out', async t => { const markdown = renderEstimate(await estimateOf({}, [])); - t.true(markdown.includes('contributed no files')); + t.true(markdown.includes('are not checked out')); t.true(markdown.includes('--clone')); }); +test('a checked-out repo that matches no file is not told to clone', async t => { + // Present on disk, but the pack scopes to src/**/*.ts and none of it is. + const estimate = await estimateOf({}, [ + {path: 'README.md', content: '# nothing to audit'}, + ]); + t.is(estimate.repos[0]?.files, 0); + t.is(estimate.repos[0]?.filesPresent, 1); + + const markdown = renderEstimate(estimate); + t.true(markdown.includes('no file matched')); + t.false(markdown.includes('--clone')); +}); + test('warns about missing packs, unparseable packs, and target errors', t => { const markdown = renderEstimate({ repos: [ @@ -554,6 +671,7 @@ test('warns about missing packs, unparseable packs, and target errors', t => { repo: 'my-org/a', packs: ['p'], files: 3, + filesPresent: 3, requests: 1, tokens: 100, durationMs: 1000, @@ -589,3 +707,34 @@ test('says so when no repositories resolved', async t => { t.true(markdown.includes('No repositories resolved')); t.false(markdown.includes('| Repository |')); }); + +test('calibrate survives a record with no prompt tokens', async t => { + // A legacy or truncated record: requests and duration, but no token counts. + // The per-token term has nothing to divide by, so all of the measured cost + // stays fixed rather than becoming NaN and poisoning every figure. + const calibration = calibrate([ + record( + {requests: 2, durationMs: 8000, promptTokens: 0, outputTokens: 0}, + 2, + ), + ]); + t.is(calibration.msPerRequest, 4000); + t.is(calibration.msPerPromptToken, 0); + + const estimate = await estimateRun( + config(), + { + files: repoFiles(FILES), + packs: packLoader({packs: [pack('p')], errors: []}), + records: [ + record( + {requests: 2, durationMs: 8000, promptTokens: 0, outputTokens: 0}, + 2, + ), + ], + }, + OPTIONS, + ); + t.true(Number.isFinite(estimate.totals.durationMs)); + t.true(estimate.totals.durationMs > 0); +}); diff --git a/source/run/estimate.ts b/source/run/estimate.ts index f73a9c7..722d1ce 100644 --- a/source/run/estimate.ts +++ b/source/run/estimate.ts @@ -28,8 +28,17 @@ const CALIBRATION_WINDOW = 10; /** The per-request figures an estimate is built from. */ export interface Calibration { - /** Wall-clock milliseconds one model request takes. */ + /** + * Fixed wall-clock milliseconds a request costs regardless of size — process + * start, model load, the round trip. + */ msPerRequest: number; + /** + * Marginal milliseconds per prompt token. Runtime is dominated by prompt size + * on a local model, so an estimate for a config far larger than the recorded + * ones would be wrong by roughly that ratio without this term. + */ + msPerPromptToken: number; /** Requests per pack pass — above 1 when auto-fix retries are common. */ requestsPerPass: number; /** Tokens the model returns per request. */ @@ -40,10 +49,13 @@ export interface Calibration { /** * Used until a run has been recorded: a minute-ish per request on a local - * model, and roughly one pass in ten needing the auto-fix retry. + * model, and roughly one pass in ten needing the auto-fix retry. The two + * duration terms sum to that 45s at a 2,500-token prompt — a typical single + * pack pass — and diverge from it as prompts get larger or smaller. */ const DEFAULT_CALIBRATION: Calibration = { - msPerRequest: 45_000, + msPerRequest: 20_000, + msPerPromptToken: 10, requestsPerPass: 1.1, outputTokensPerRequest: 700, samples: 0, @@ -56,6 +68,12 @@ export interface RepoEstimate { packs: string[]; /** Distinct files at least one pack will send to the model. */ files: number; + /** + * Files found in the checkout before `applies_to` scoping. Zero means the + * repo is not checked out; non-zero with `files` at zero means it is present + * but no pack matches anything in it. + */ + filesPresent: number; /** Model requests, retries included. */ requests: number; tokens: number; @@ -104,7 +122,83 @@ export interface EstimateOptions { /** Approximate the token count of a piece of prompt or completion text. */ export function estimateTokens(text: string): number { - return Math.ceil(text.length / CHARS_PER_TOKEN); + return tokensFromChars(text.length); +} + +/** + * The same approximation for a character count already accumulated — the audit + * loop totals characters across retries rather than holding every prompt. + */ +export function tokensFromChars(chars: number): number { + return Math.ceil(chars / CHARS_PER_TOKEN); +} + +/** One record reduced to per-request cost against per-request prompt size. */ +interface DurationPoint { + tokens: number; + ms: number; +} + +/** + * Least-squares fit of ms = msPerRequest + msPerPromptToken * tokens. + * + * Returns null when the records cannot separate the two terms — fewer than two + * of them, or every run the same prompt size — and when the fit comes back + * nonsensical (a negative term, which noise across few samples can produce). + * The caller falls back to splitting the observed average instead. + */ +function fitDurationTerms( + points: DurationPoint[], +): Pick | null { + if (points.length < 2) { + return null; + } + + const meanTokens = + points.reduce((total, point) => total + point.tokens, 0) / points.length; + const meanMs = + points.reduce((total, point) => total + point.ms, 0) / points.length; + + let variance = 0; + let covariance = 0; + for (const point of points) { + const spread = point.tokens - meanTokens; + variance += spread * spread; + covariance += spread * (point.ms - meanMs); + } + if (variance === 0) { + return null; + } + + const msPerPromptToken = covariance / variance; + const msPerRequest = meanMs - msPerPromptToken * meanTokens; + if (msPerPromptToken < 0 || msPerRequest < 0) { + return null; + } + return {msPerRequest, msPerPromptToken}; +} + +/** + * Split one averaged observation across the two duration terms, keeping the + * proportion the built-in defaults imply at that prompt size. Used when the + * records cannot support a fit: the magnitude is measured even though the + * shape is assumed, which beats attributing all of it to either term. + */ +function splitDurationTerms( + msPerRequest: number, + tokensPerRequest: number, +): Pick { + if (tokensPerRequest <= 0) { + return {msPerRequest, msPerPromptToken: 0}; + } + const shape = + DEFAULT_CALIBRATION.msPerRequest + + DEFAULT_CALIBRATION.msPerPromptToken * tokensPerRequest; + const fixedShare = DEFAULT_CALIBRATION.msPerRequest / shape; + return { + msPerRequest: msPerRequest * fixedShare, + msPerPromptToken: (msPerRequest * (1 - fixedShare)) / tokensPerRequest, + }; } /** @@ -143,15 +237,28 @@ export function calibrate(records: RunRecord[]): Calibration { let passes = 0; let durationMs = 0; let outputTokens = 0; + let promptTokens = 0; + const points: DurationPoint[] = []; for (const sample of usable) { requests += sample.usage.requests; durationMs += sample.usage.durationMs; outputTokens += sample.usage.outputTokens; + promptTokens += sample.usage.promptTokens; passes += sample.passes; + points.push({ + tokens: sample.usage.promptTokens / sample.usage.requests, + ms: sample.usage.durationMs / sample.usage.requests, + }); } + // Prefer separating the fixed and per-token terms from the spread across + // records; fall back to splitting the pooled average when they cannot. + const duration = + fitDurationTerms(points) ?? + splitDurationTerms(durationMs / requests, promptTokens / requests); + return { - msPerRequest: durationMs / requests, + ...duration, requestsPerPass: passes > 0 ? Math.max(1, requests / passes) @@ -202,16 +309,26 @@ export async function estimateRun( } const requests = Math.round(packs.length * calibration.requestsPerPass); + // A retry resends the prompt, so both sides scale with requests. + const promptTokensSent = Math.round( + promptTokens * calibration.requestsPerPass, + ); repos.push({ repo: target.repo, packs: packs.map(pack => pack.manifest.name), files: audited.size, + filesPresent: files.length, requests, - // A retry resends the prompt, so both sides scale with requests. tokens: - Math.round(promptTokens * calibration.requestsPerPass) + + promptTokensSent + Math.round(requests * calibration.outputTokensPerRequest), - durationMs: Math.round(requests * calibration.msPerRequest), + // Fixed per-request cost plus the part that tracks prompt size, so a + // config far larger than the recorded runs is not priced as if it were + // the same size. + durationMs: Math.round( + requests * calibration.msPerRequest + + promptTokensSent * calibration.msPerPromptToken, + ), missingPacks: missing, }); } @@ -253,7 +370,9 @@ function formatTokens(value: number): string { /** Wall-clock, rounded to the unit an operator actually schedules in. */ function formatDuration(ms: number): string { const seconds = Math.round(ms / 1000); - if (seconds < 90) { + // Hand over at a minute, not 90s: rounding the minutes figure from 90s + // upward would otherwise make "1 minute(s)" unreachable. + if (seconds < 60) { return `${seconds} second(s)`; } const minutes = Math.round(seconds / 60); @@ -273,10 +392,21 @@ function repoRow(repo: RepoEstimate): string { function caveats(estimate: AuditEstimate): string[] { const notes: string[] = []; - const empty = estimate.repos.filter(repo => repo.files === 0); - if (empty.length > 0) { + const absent = estimate.repos.filter(repo => repo.filesPresent === 0); + if (absent.length > 0) { + notes.push( + `> ⚠️ ${absent.length} repo(s) are not checked out, so their cost is understated — check them out under the workspace, or pass --clone: ${absent.map(repo => repo.repo).join(', ')}`, + ); + } + + // Present but scoped away: nothing to clone, and a pack is pointed at a repo + // it cannot see — the more useful of the two warnings. + const unmatched = estimate.repos.filter( + repo => repo.filesPresent > 0 && repo.files === 0, + ); + if (unmatched.length > 0) { notes.push( - `> ⚠️ ${empty.length} repo(s) contributed no files, so their cost is understated — check them out under the workspace, or pass --clone: ${empty.map(repo => repo.repo).join(', ')}`, + `> ⚠️ ${unmatched.length} repo(s) are checked out but no file matched their packs' applies_to, so they will audit nothing: ${unmatched.map(repo => repo.repo).join(', ')}`, ); } for (const repo of estimate.repos) { From 13489521a97e8394f7c10ce9a59ed949a66149d9 Mon Sep 17 00:00:00 2001 From: addyCooks Date: Wed, 2 Sep 2026 01:46:21 +0530 Subject: [PATCH 3/3] fix(estimate): source the checkout probe independently of applies_to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix did not work. `filesPresent` came from `files.length`, but that read is already scoped: const files = await deps.files.read(repoDir, unionPatterns(packs)); fsRepoFiles.read filters by the patterns it is given, so the count was post-applies_to too. A checked-out TypeScript repo with a single `**/*.py` pack still produced zero, so it was still reported as not checked out and still told to clone what it already had. Only the cases where the union is broader than one pack's own matching — a multi-pack target where another pack matched, or an empty `paths` making the union everything — happened to work. `RepoEstimate.checkedOut` replaces the count with what the warning actually needs to know, and the probe is a second read with no patterns. That read runs only when the scoped one came back empty: with files in hand the repo is obviously present, so the extra read is skipped on every path where it could not change the answer. The test could not see any of this. The repoFiles stub ignored its patterns argument and returned everything, modelling a read production never performs, so a repo that matched nothing still arrived with files. The stub now applies patterns exactly as fsRepoFiles does. Both halves were needed: with the old stub, the broken implementation passes the whole suite, including the test written to catch it. Covers the case from review directly — one pack scoped to a language the repo does not contain — plus the absent-checkout counterpart, and a test pinning that the unscoped read is skipped when the scoped one found files. --- source/run/estimate.spec.ts | 95 +++++++++++++++++++++++++++++++++++-- source/run/estimate.ts | 22 ++++++--- 2 files changed, 106 insertions(+), 11 deletions(-) diff --git a/source/run/estimate.spec.ts b/source/run/estimate.spec.ts index 447d9f1..ef01dcd 100644 --- a/source/run/estimate.spec.ts +++ b/source/run/estimate.spec.ts @@ -2,6 +2,7 @@ import test from 'ava'; import type {SentinelConfig} from '../config/types.js'; import type {RunRecord, RunUsage} from '../observe/types.js'; import type {SourceFile} from '../prompt/types.js'; +import {matchesGlob} from '../rule-packs/glob.js'; import type {RulePack} from '../rule-packs/types.js'; import type {PrepareResult} from './clone.js'; import { @@ -53,10 +54,22 @@ function packLoader(loaded: LoadedPacks): PackLoader { }; } +/** + * Stands in for a checked-out repository. It must apply the patterns it is + * given exactly as fsRepoFiles does — a stub that returns everything models a + * read production never performs, and cannot tell an absent checkout from a + * present one that nothing matched. + */ function repoFiles(files: SourceFile[]): RepoFiles { return { - async read(): Promise { - return files; + async read(_repoDir: string, patterns: string[]): Promise { + // An empty pattern list means the whole repository. + if (patterns.length === 0) { + return files; + } + return files.filter(file => + patterns.some(pattern => matchesGlob(pattern, file.path)), + ); }, async readText(): Promise { return null; @@ -657,7 +670,7 @@ test('a checked-out repo that matches no file is not told to clone', async t => {path: 'README.md', content: '# nothing to audit'}, ]); t.is(estimate.repos[0]?.files, 0); - t.is(estimate.repos[0]?.filesPresent, 1); + t.true(estimate.repos[0]?.checkedOut); const markdown = renderEstimate(estimate); t.true(markdown.includes('no file matched')); @@ -671,7 +684,7 @@ test('warns about missing packs, unparseable packs, and target errors', t => { repo: 'my-org/a', packs: ['p'], files: 3, - filesPresent: 3, + checkedOut: true, requests: 1, tokens: 100, durationMs: 1000, @@ -738,3 +751,77 @@ test('calibrate survives a record with no prompt tokens', async t => { t.true(Number.isFinite(estimate.totals.durationMs)); t.true(estimate.totals.durationMs > 0); }); + +// The case from review: one pack scoped to a language the repo does not +// contain. Both the scoped read and the audited set come back empty, so the +// only thing that separates this from an un-cloned repo is a read that ignores +// the patterns. Getting it wrong sends the operator to clone what they have. +test('a single pack matching nothing is not mistaken for a missing checkout', async t => { + const python = pack('py'); + python.manifest.appliesTo = {paths: ['**/*.py'], languages: ['python']}; + + const estimate = await estimateRun( + config({targets: [{repo: 'my-org/a', rulePacks: ['py']}]}), + { + files: repoFiles(FILES), // a TypeScript repo, checked out + packs: packLoader({packs: [python], errors: []}), + }, + OPTIONS, + ); + + t.is(estimate.repos[0]?.files, 0); + t.true(estimate.repos[0]?.checkedOut); + + const markdown = renderEstimate(estimate); + t.true(markdown.includes('no file matched')); + t.false(markdown.includes('--clone')); +}); + +test('an absent checkout is still told to clone', async t => { + // Same shape, but nothing on disk at all: the unscoped read is empty too. + const python = pack('py'); + python.manifest.appliesTo = {paths: ['**/*.py'], languages: ['python']}; + + const estimate = await estimateRun( + config({targets: [{repo: 'my-org/a', rulePacks: ['py']}]}), + { + files: repoFiles([]), + packs: packLoader({packs: [python], errors: []}), + }, + OPTIONS, + ); + + t.false(estimate.repos[0]?.checkedOut); + const markdown = renderEstimate(estimate); + t.true(markdown.includes('are not checked out')); + t.true(markdown.includes('--clone')); +}); + +test('the unscoped read is skipped when the scoped one found files', async t => { + // The extra read exists only to disambiguate an empty result, so it must not + // cost anything on the common path. + const reads: string[][] = []; + const counting: RepoFiles = { + async read(_repoDir: string, patterns: string[]): Promise { + reads.push(patterns); + return patterns.length === 0 + ? FILES + : FILES.filter(file => + patterns.some(pattern => matchesGlob(pattern, file.path)), + ); + }, + async readText(): Promise { + return null; + }, + }; + + const estimate = await estimateRun( + config(), + {files: counting, packs: packLoader({packs: [pack('p')], errors: []})}, + OPTIONS, + ); + + t.true(estimate.repos[0]?.checkedOut); + t.is(reads.length, 1); + t.deepEqual(reads[0], ['src/**/*.ts']); +}); diff --git a/source/run/estimate.ts b/source/run/estimate.ts index 722d1ce..ca6b374 100644 --- a/source/run/estimate.ts +++ b/source/run/estimate.ts @@ -69,11 +69,12 @@ export interface RepoEstimate { /** Distinct files at least one pack will send to the model. */ files: number; /** - * Files found in the checkout before `applies_to` scoping. Zero means the - * repo is not checked out; non-zero with `files` at zero means it is present - * but no pack matches anything in it. + * Whether the repository is present under the workspace at all. Separate + * from `files`, which is a count after `applies_to` scoping: a checked-out + * repo that no pack matches also has zero files, and cloning will not fix + * it, so the two need different advice. */ - filesPresent: number; + checkedOut: boolean; /** Model requests, retries included. */ requests: number; tokens: number; @@ -297,6 +298,13 @@ export async function estimateRun( const {packs, missing} = selectPacks(loaded.packs, target.rulePacks); const files = await deps.files.read(repoDir, unionPatterns(packs)); + // `read` applies the patterns it is given, so this count alone cannot tell + // an absent checkout from a present one that nothing matched. Only when it + // comes back empty is an unscoped read worth its cost, and only then can + // it change which warning the operator gets. + const checkedOut = + files.length > 0 || (await deps.files.read(repoDir, [])).length > 0; + // Build the real prompts: what the model is sent is what we count. const audited = new Set(); let promptTokens = 0; @@ -317,7 +325,7 @@ export async function estimateRun( repo: target.repo, packs: packs.map(pack => pack.manifest.name), files: audited.size, - filesPresent: files.length, + checkedOut, requests, tokens: promptTokensSent + @@ -392,7 +400,7 @@ function repoRow(repo: RepoEstimate): string { function caveats(estimate: AuditEstimate): string[] { const notes: string[] = []; - const absent = estimate.repos.filter(repo => repo.filesPresent === 0); + const absent = estimate.repos.filter(repo => !repo.checkedOut); if (absent.length > 0) { notes.push( `> ⚠️ ${absent.length} repo(s) are not checked out, so their cost is understated — check them out under the workspace, or pass --clone: ${absent.map(repo => repo.repo).join(', ')}`, @@ -402,7 +410,7 @@ function caveats(estimate: AuditEstimate): string[] { // Present but scoped away: nothing to clone, and a pack is pointed at a repo // it cannot see — the more useful of the two warnings. const unmatched = estimate.repos.filter( - repo => repo.filesPresent > 0 && repo.files === 0, + repo => repo.checkedOut && repo.files === 0, ); if (unmatched.length > 0) { notes.push(