Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions skills/llm-council/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ Karpathy's LLM Council pattern, provider-agnostic. dair-academy's version hardco

## Three phases

1. **Independent**: each model answers in parallel
2. **Ranking**: each model ranks anonymized peer responses
1. **Independent**: each model answers in parallel by default
2. **Ranking**: each model ranks anonymized peer responses in parallel by default
3. **Synthesis**: chairman model reads all responses + rankings → final answer

`--sequential` overrides the default parallel behavior for phases 1 and 2, running their model calls one at a time.

## Provider config

Provider chosen via env. First-match wins:
Expand All @@ -42,11 +44,22 @@ Default model rosters per provider live in `scripts/council.js` and can be overr
## Commands

```
node $SKILL_ROOT/scripts/council.js run "<query>" [--models id1,id2,id3] [--chairman id] [--provider <name>] [--wiki <slug>]
node $SKILL_ROOT/scripts/council.js run "<query>" [--models id1,id2,id3] [--chairman id] [--provider <name>] [--wiki <slug>]\
[--max-tokens N] [--timeout ms] [--max-retries N] [--sequential] [--reasoning-effort low|medium|high]
node $SKILL_ROOT/scripts/council.js providers
node $SKILL_ROOT/scripts/council.js show <session-id>
```

### Runtime options

| Flag | Default | Description |
|------|---------|-------------|
| `--max-tokens` | 4000 | Max output tokens per model call. Bump to 16000+ for reasoning models. |
| `--timeout` | 120000 | HTTP request timeout in ms. Bump to 300000+ for slow endpoints (NVIDIA NIM). |
| `--max-retries` | 1 | Retry count on 429/5xx and connection errors. Exponential backoff (2s, 4s, 8s...). |
| `--sequential` | false | Run model calls one at a time instead of in parallel. Use when free endpoints reject concurrent requests. |
| `--reasoning-effort` | — | low\|medium\|high passed as `reasoning.effort` (OpenAI-compat) or `output_config.effort` (Anthropic adaptive thinking). |

`--wiki <slug>` writes the full transcript to `<wiki>/derived/council/<session-id>.md` and registers it via `wiki-cli.js page` so it shows in FTS5 search.

## Output
Expand Down
103 changes: 92 additions & 11 deletions skills/llm-council/scripts/council.js
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,11 @@ function pickProvider(arg) {
return null;
}

function postJSON(urlStr, body, headers, timeoutMs = 120000) {
// Runtime tuning knobs that can be overridden via CLI flags in `cmdRun`.
// Defaults preserve previous hard-coded behavior.
const RUN_OPTS = { max_tokens: 4000, timeout_ms: 120000, max_retries: 1, sequential: false, reasoning_effort: null };

function postJSON(urlStr, body, headers, timeoutMs = RUN_OPTS.timeout_ms) {
return new Promise((resolve, reject) => {
const url = new URL(urlStr);
const data = JSON.stringify(body);
Expand All @@ -78,16 +82,41 @@ function postJSON(urlStr, body, headers, timeoutMs = 120000) {
});
}

async function postJSONWithRetry(urlStr, body, headers) {
const delay = attempt => new Promise(r => setTimeout(r, 2000 * Math.pow(2, attempt - 1)));
let res;
let lastError;
for (let attempt = 0; ; attempt++) {
try {
res = await postJSON(urlStr, body, headers);
} catch (e) {
// Connection-level error (ETIMEDOUT, ECONNRESET)
lastError = e;
if (attempt >= RUN_OPTS.max_retries) return { error: lastError };
await delay(attempt + 1);
continue;
}
if (res.status < 500 && res.status !== 429) return { res };
if (attempt >= RUN_OPTS.max_retries) return { res };
await delay(attempt + 1);
}
}

async function callOpenAICompat(provider, model, system, user) {
const start = Date.now();
const url = `${provider.baseUrl}/chat/completions`;
const res = await postJSON(url, {
const payload = {
model,
messages: [{ role: 'system', content: system }, { role: 'user', content: user }],
max_tokens: 4000,
max_tokens: RUN_OPTS.max_tokens,
temperature: 1,
}, { Authorization: `Bearer ${process.env[provider.envKey]}` });
};
if (RUN_OPTS.reasoning_effort) payload.reasoning = { effort: RUN_OPTS.reasoning_effort };
const authHeaders = { Authorization: `Bearer ${process.env[provider.envKey]}` };

const { res, error } = await postJSONWithRetry(url, payload, authHeaders);
const elapsed = Date.now() - start;
if (error) return { success: false, content: `[ERROR connection: ${error.code || error.message}]`, model, latency_ms: elapsed };
if (res.status >= 400) return { success: false, content: `[ERROR ${res.status}: ${res.body.slice(0, 300)}]`, model, latency_ms: elapsed };
let data;
try { data = JSON.parse(res.body); } catch (e) { return { success: false, content: `[parse-error]`, model, latency_ms: elapsed }; }
Expand All @@ -98,16 +127,24 @@ async function callOpenAICompat(provider, model, system, user) {
async function callAnthropic(provider, model, system, user) {
const start = Date.now();
const url = `${provider.baseUrl}/v1/messages`;
const res = await postJSON(url, {
const payload = {
model,
max_tokens: 4000,
max_tokens: RUN_OPTS.max_tokens,
system,
messages: [{ role: 'user', content: user }],
}, {
};
if (RUN_OPTS.reasoning_effort) {
payload.thinking = { type: 'adaptive' };
payload.output_config = { effort: RUN_OPTS.reasoning_effort };
}
const authHeaders = {
'x-api-key': process.env[provider.envKey],
'anthropic-version': '2023-06-01',
});
};

const { res, error } = await postJSONWithRetry(url, payload, authHeaders);
const elapsed = Date.now() - start;
if (error) return { success: false, content: `[ERROR connection: ${error.code || error.message}]`, model, latency_ms: elapsed };
if (res.status >= 400) return { success: false, content: `[ERROR ${res.status}: ${res.body.slice(0, 300)}]`, model, latency_ms: elapsed };
let data;
try { data = JSON.parse(res.body); } catch { return { success: false, content: '[parse-error]', model, latency_ms: elapsed }; }
Expand Down Expand Up @@ -164,6 +201,39 @@ async function cmdRun(args) {
const provider = PROVIDERS[providerName];
if (!provider.baseUrl) { console.error(`provider ${providerName} requires LLM_COUNCIL_BASE_URL`); process.exit(2); }

function parseIntSafe(val, name) {
const n = parseInt(val, 10);
if (isNaN(n) || n <= 0 || n !== Math.floor(n)) {
console.error(`Invalid --${name}: ${val} (must be a positive integer)`);
process.exit(2);
}
return n;
}
if (args['max-tokens']) RUN_OPTS.max_tokens = parseIntSafe(args['max-tokens'], 'max-tokens');
if (args.timeout) RUN_OPTS.timeout_ms = parseIntSafe(args.timeout, 'timeout');
if (args['max-retries']) RUN_OPTS.max_retries = parseIntSafe(args['max-retries'], 'max-retries');
if (args.sequential) RUN_OPTS.sequential = true;
if (args['reasoning-effort']) {
const effort = args['reasoning-effort'];
if (!['low', 'medium', 'high'].includes(effort)) {
console.error(`Invalid --reasoning-effort: ${effort} (must be low, medium, or high)`);
process.exit(2);
}
RUN_OPTS.reasoning_effort = effort;
}

// Run a list of async callables either in parallel (default) or sequentially.
// Sequential mode avoids concurrent-request limits on free NIM/OpenRouter endpoints.
async function runCalls(callables) {
if (!RUN_OPTS.sequential) return Promise.allSettled(callables.map(fn => fn()));
const results = [];
for (const fn of callables) {
try { results.push({ status: 'fulfilled', value: await fn() }); }
catch (e) { results.push({ status: 'rejected', reason: e }); }
}
return results;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

const models = (args.models ? String(args.models).split(',') : provider.defaultModels).filter(Boolean);
const chairman = args.chairman || provider.defaultChairman;
if (!models.length) { console.error('no models — pass --models'); process.exit(2); }
Expand All @@ -182,7 +252,7 @@ async function cmdRun(args) {

// Phase 1
const sysIndep = 'You are participating in an LLM council deliberation. Provide your best, most thoughtful response to the query. Be comprehensive but focused.';
const phase1Settled = await Promise.allSettled(models.map(m => provider.call(provider, m, sysIndep, query)));
const phase1Settled = await runCalls(models.map(m => () => provider.call(provider, m, sysIndep, query)));
const phase1Entries = phase1Settled.map((s, i) => settledToEntry(models[i], s));
const phase1 = Object.fromEntries(models.map((m, i) => [m, phase1Entries[i]]));
fs.writeFileSync(path.join(sessionDir, 'phase1_responses.json'), JSON.stringify(phase1, null, 2));
Expand All @@ -193,7 +263,7 @@ async function cmdRun(args) {
const anon = models.map(m => `=== Response ${labelOf[m]} ===\n${phase1[m].content}`).join('\n\n');
const sysRank = (own) => `You are ranking AI responses objectively. Your own response is labeled '${own}'.`;
const userRank = `QUERY:\n${query}\n\nRESPONSES:\n${anon}\n\nRank from BEST to WORST. Format:\nRANKINGS:\n1. [Letter] - [reason]\n2. [Letter] - [reason]\n...`;
const phase2Settled = await Promise.allSettled(models.map(m => provider.call(provider, m, sysRank(labelOf[m]), userRank)));
const phase2Settled = await runCalls(models.map(m => () => provider.call(provider, m, sysRank(labelOf[m]), userRank)));
const phase2Entries = phase2Settled.map((s, i) => settledToEntry(models[i], s));
const phase2 = { label_of: labelOf, rankings: Object.fromEntries(models.map((m, i) => [m, phase2Entries[i]])) };
fs.writeFileSync(path.join(sessionDir, 'phase2_rankings.json'), JSON.stringify(phase2, null, 2));
Expand Down Expand Up @@ -267,8 +337,19 @@ function cmdShow(args) {
function usage() {
console.error(`Usage:
council.js run "<query>" [--models id1,id2,id3] [--chairman id] [--provider name] [--wiki slug]
[--max-tokens N] [--timeout ms] [--max-retries N] [--sequential]
[--reasoning-effort low|medium|high]
council.js providers
council.js show <session-id>`);
council.js show <session-id>

Options:
--max-tokens Max output tokens per model call (default 4000; bump to 16000+ for reasoning models)
--timeout HTTP request timeout in ms (default 120000; bump to 300000+ for slow NIM endpoints)
--max-retries Retry count on connection errors and 429/5xx (default 1; exponential backoff 2s, 4s, ...)
--sequential Run model calls one at a time instead of in parallel (use when free endpoints
like NVIDIA NIM reject concurrent requests with ETIMEDOUT / 429)
--reasoning-effort Pass reasoning effort to OpenAI-compat payload as reasoning.effort
(low|medium|high) — for o1/o3/DeepSeek/Claude thinking models`);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
process.exit(1);
}

Expand Down