Skip to content

Commit 9b2e0b8

Browse files
Copilotlpcox
Copilot
and
lpcox
authored
feat(api-proxy): add /reflect endpoint for dynamic provider and model discovery (#2253)
* Initial plan * feat: add reflection endpoint to api-proxy sidecar Add GET /reflect on port 10000 (management port) that returns the list of configured API proxy endpoints along with the models supported by each endpoint. - fetchJson: helper to fetch and parse JSON responses from provider APIs - extractModelIds: normalise OpenAI/Anthropic/Copilot {data:[{id}]} and Gemini {models:[{name}]} formats into sorted string arrays - cachedModels / resetModelCacheState: in-memory model cache populated at startup, with reset helper for test isolation - fetchStartupModels: fetches model lists from all configured providers concurrently at startup (alongside validateApiKeys) - reflectEndpoints: builds the reflection payload with per-endpoint configured status, base_url, port, models_url and cached models - handleManagementEndpoint: extended to serve GET /reflect - onListenerReady: triggers fetchStartupModels in addition to validateApiKeys - All new functions exported and covered by unit tests (306 passing) Agent-Logs-Url: https://github.com/github/gh-aw-firewall/sessions/36f4f63e-88fd-493f-a600-7fab58452dd6 * feat: improve reflection endpoint based on review feedback - fetchJson: add debug-level logging for network errors and timeouts to aid operator diagnostics during model discovery - extractModelIds: extract GEMINI_MODEL_NAME_PREFIX constant and use startsWith/slice for prefix stripping (clearer than regex) - Add test for Gemini model names without the models/ prefix Agent-Logs-Url: https://github.com/github/gh-aw-firewall/sessions/36f4f63e-88fd-493f-a600-7fab58452dd6 * fix: address fetchJson hang and Copilot BYOK model-fetch issues - fetchJson: add res.on('close') handler so the Promise always settles when the upstream connection drops mid-body without emitting 'end' or 'error', preventing modelFetchComplete from hanging indefinitely - fetchStartupModels: gate Copilot /models fetch exclusively on copilotGithubToken (GitHub OAuth); skip when only COPILOT_API_KEY (BYOK) is present — consistent with validateApiKeys behaviour where BYOK-only mode is documented as having no probe endpoint - Tests: add cases for res.close mid-body drop and BYOK-only skip Agent-Logs-Url: https://github.com/github/gh-aw-firewall/sessions/cd6fa904-5c17-4a7e-bcd6-0f0f44061d0e Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: lpcox <15877973+lpcox@users.noreply.github.com>
1 parent 2f5cc71 commit 9b2e0b8

2 files changed

Lines changed: 591 additions & 3 deletions

File tree

‎containers/api-proxy/server.js‎

Lines changed: 278 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1145,6 +1145,273 @@ function httpProbe(url, opts, timeoutMs) {
11451145
});
11461146
}
11471147

1148+
/**
1149+
* Make an HTTPS/HTTP request through the proxy and return parsed JSON response.
1150+
* Returns null on any error, non-2xx status, or parse failure.
1151+
*
1152+
* @param {string} url
1153+
* @param {{ method: string, headers: Record<string,string> }} opts
1154+
* @param {number} timeoutMs
1155+
* @returns {Promise<object|null>}
1156+
*/
1157+
function fetchJson(url, opts, timeoutMs) {
1158+
return new Promise((resolve) => {
1159+
let parsed;
1160+
try {
1161+
parsed = new URL(url);
1162+
} catch {
1163+
resolve(null);
1164+
return;
1165+
}
1166+
const isHttps = parsed.protocol === 'https:';
1167+
const mod = isHttps ? https : http;
1168+
const reqOpts = {
1169+
hostname: parsed.hostname,
1170+
port: parsed.port || (isHttps ? 443 : 80),
1171+
path: parsed.pathname + parsed.search,
1172+
method: opts.method,
1173+
headers: { ...opts.headers },
1174+
...(isHttps && proxyAgent ? { agent: proxyAgent } : {}),
1175+
timeout: timeoutMs,
1176+
};
1177+
1178+
let settled = false;
1179+
const resolveOnce = (value) => {
1180+
if (settled) return;
1181+
settled = true;
1182+
resolve(value);
1183+
};
1184+
1185+
const req = mod.request(reqOpts, (res) => {
1186+
if (res.statusCode < 200 || res.statusCode >= 300) {
1187+
res.resume();
1188+
resolveOnce(null);
1189+
return;
1190+
}
1191+
const chunks = [];
1192+
res.on('data', (chunk) => chunks.push(chunk));
1193+
res.on('end', () => {
1194+
try {
1195+
resolveOnce(JSON.parse(Buffer.concat(chunks).toString()));
1196+
} catch {
1197+
resolveOnce(null);
1198+
}
1199+
});
1200+
res.on('error', (err) => {
1201+
logRequest('debug', 'fetch_json_error', { url: sanitizeForLog(url), error: String(err && err.message ? err.message : err) });
1202+
resolveOnce(null);
1203+
});
1204+
// Guard against connection drops mid-body that never emit 'end' or 'error'
1205+
res.on('close', () => resolveOnce(null));
1206+
});
1207+
1208+
req.on('timeout', () => {
1209+
const err = new Error(`fetchJson timed out after ${timeoutMs}ms`);
1210+
logRequest('debug', 'fetch_json_timeout', { url: sanitizeForLog(url), timeout_ms: timeoutMs });
1211+
req.destroy(err);
1212+
});
1213+
req.on('error', (err) => {
1214+
logRequest('debug', 'fetch_json_error', { url: sanitizeForLog(url), error: String(err && err.message ? err.message : err) });
1215+
resolveOnce(null);
1216+
});
1217+
req.end();
1218+
});
1219+
}
1220+
1221+
/**
1222+
* Prefix used by the Gemini models API in model name fields.
1223+
* Example: { name: "models/gemini-1.5-pro" } → "gemini-1.5-pro"
1224+
*/
1225+
const GEMINI_MODEL_NAME_PREFIX = 'models/';
1226+
1227+
/**
1228+
* Extract model IDs from a provider API response.
1229+
* Handles:
1230+
* - OpenAI / Anthropic / Copilot format: { data: [{ id }, ...] }
1231+
* - Gemini format: { models: [{ name: "models/gemini-1.5-pro" }, ...] }
1232+
*
1233+
* @param {object|null} json - Parsed API response
1234+
* @returns {string[]|null} Sorted array of model IDs, or null if unavailable
1235+
*/
1236+
function extractModelIds(json) {
1237+
if (!json || typeof json !== 'object') return null;
1238+
1239+
// OpenAI / Anthropic / Copilot format: { data: [{ id: "..." }, ...] }
1240+
if (Array.isArray(json.data)) {
1241+
const ids = json.data
1242+
.map((m) => m && (m.id || m.name))
1243+
.filter(Boolean);
1244+
return ids.length > 0 ? ids.sort() : null;
1245+
}
1246+
1247+
// Gemini format: { models: [{ name: "models/gemini-1.5-pro", ... }, ...] }
1248+
if (Array.isArray(json.models)) {
1249+
const ids = json.models
1250+
.map((m) => m && m.name && m.name.startsWith(GEMINI_MODEL_NAME_PREFIX)
1251+
? m.name.slice(GEMINI_MODEL_NAME_PREFIX.length)
1252+
: (m && m.name) || null)
1253+
.filter(Boolean);
1254+
return ids.length > 0 ? ids.sort() : null;
1255+
}
1256+
1257+
return null;
1258+
}
1259+
1260+
/**
1261+
* Cache for available models per provider, populated at startup by fetchStartupModels.
1262+
* null = not yet fetched or fetch failed for this provider.
1263+
* @type {Record<string, string[]|null>}
1264+
*/
1265+
const cachedModels = {};
1266+
1267+
/** Set to true once fetchStartupModels() has run (regardless of success). */
1268+
let modelFetchComplete = false;
1269+
1270+
/** Reset model cache state (used in tests). */
1271+
function resetModelCacheState() {
1272+
for (const key of Object.keys(cachedModels)) {
1273+
delete cachedModels[key];
1274+
}
1275+
modelFetchComplete = false;
1276+
}
1277+
1278+
/**
1279+
* Fetch available models for each configured provider and cache them.
1280+
* Called at startup alongside key validation.
1281+
*
1282+
* Accepts the same override map as validateApiKeys() so tests can inject
1283+
* custom keys and targets without touching process.env.
1284+
*
1285+
* @param {object} [overrides={}] - Optional key/target overrides (used in tests)
1286+
*/
1287+
async function fetchStartupModels(overrides = {}) {
1288+
const ov = (key, fallback) => key in overrides ? overrides[key] : fallback;
1289+
const openaiKey = ov('openaiKey', OPENAI_API_KEY);
1290+
const openaiTarget = ov('openaiTarget', OPENAI_API_TARGET);
1291+
const anthropicKey = ov('anthropicKey', ANTHROPIC_API_KEY);
1292+
const anthropicTarget = ov('anthropicTarget', ANTHROPIC_API_TARGET);
1293+
const copilotGithubToken = ov('copilotGithubToken', COPILOT_GITHUB_TOKEN);
1294+
const copilotAuthToken = ov('copilotAuthToken', COPILOT_AUTH_TOKEN);
1295+
const copilotTarget = ov('copilotTarget', COPILOT_API_TARGET);
1296+
const copilotIntegrationId = ov('copilotIntegrationId', COPILOT_INTEGRATION_ID);
1297+
const geminiKey = ov('geminiKey', GEMINI_API_KEY);
1298+
const geminiTarget = ov('geminiTarget', GEMINI_API_TARGET);
1299+
const TIMEOUT_MS = ov('timeoutMs', 10_000);
1300+
1301+
const fetches = [];
1302+
1303+
if (openaiKey) {
1304+
fetches.push(
1305+
fetchJson(`https://${openaiTarget}/v1/models`, {
1306+
method: 'GET',
1307+
headers: { 'Authorization': `Bearer ${openaiKey}` },
1308+
}, TIMEOUT_MS).then((json) => {
1309+
cachedModels.openai = extractModelIds(json);
1310+
})
1311+
);
1312+
}
1313+
1314+
if (anthropicKey) {
1315+
fetches.push(
1316+
fetchJson(`https://${anthropicTarget}/v1/models`, {
1317+
method: 'GET',
1318+
headers: { 'x-api-key': anthropicKey, 'anthropic-version': '2023-06-01' },
1319+
}, TIMEOUT_MS).then((json) => {
1320+
cachedModels.anthropic = extractModelIds(json);
1321+
})
1322+
);
1323+
}
1324+
1325+
// Only use COPILOT_GITHUB_TOKEN (GitHub OAuth) for /models — COPILOT_API_KEY (BYOK) is not
1326+
// accepted by the Copilot /models endpoint (consistent with validateApiKeys behaviour).
1327+
if (copilotGithubToken) {
1328+
fetches.push(
1329+
fetchJson(`https://${copilotTarget}/models`, {
1330+
method: 'GET',
1331+
headers: {
1332+
'Authorization': `Bearer ${copilotGithubToken}`,
1333+
'Copilot-Integration-Id': copilotIntegrationId,
1334+
},
1335+
}, TIMEOUT_MS).then((json) => {
1336+
cachedModels.copilot = extractModelIds(json);
1337+
})
1338+
);
1339+
}
1340+
1341+
if (geminiKey) {
1342+
fetches.push(
1343+
fetchJson(`https://${geminiTarget}/v1beta/models`, {
1344+
method: 'GET',
1345+
headers: { 'x-goog-api-key': geminiKey },
1346+
}, TIMEOUT_MS).then((json) => {
1347+
cachedModels.gemini = extractModelIds(json);
1348+
})
1349+
);
1350+
}
1351+
1352+
await Promise.allSettled(fetches);
1353+
modelFetchComplete = true;
1354+
}
1355+
1356+
/**
1357+
* Build the reflection response describing all proxy endpoints and their available models.
1358+
*
1359+
* The reflection endpoint allows agent harnesses to dynamically discover which
1360+
* LLM providers are configured and what models are available, enabling intelligent
1361+
* provider and model selection based on the task at hand.
1362+
*
1363+
* @returns {{ endpoints: Array<object>, models_fetch_complete: boolean }}
1364+
*/
1365+
function reflectEndpoints() {
1366+
const opencodeConfigured = !!(OPENAI_API_KEY || ANTHROPIC_API_KEY || COPILOT_AUTH_TOKEN);
1367+
return {
1368+
endpoints: [
1369+
{
1370+
provider: 'openai',
1371+
port: 10000,
1372+
base_url: 'http://api-proxy:10000',
1373+
configured: !!OPENAI_API_KEY,
1374+
models: cachedModels.openai || null,
1375+
models_url: 'http://api-proxy:10000/v1/models',
1376+
},
1377+
{
1378+
provider: 'anthropic',
1379+
port: 10001,
1380+
base_url: 'http://api-proxy:10001',
1381+
configured: !!ANTHROPIC_API_KEY,
1382+
models: cachedModels.anthropic || null,
1383+
models_url: 'http://api-proxy:10001/v1/models',
1384+
},
1385+
{
1386+
provider: 'copilot',
1387+
port: 10002,
1388+
base_url: 'http://api-proxy:10002',
1389+
configured: !!COPILOT_AUTH_TOKEN,
1390+
models: cachedModels.copilot || null,
1391+
models_url: 'http://api-proxy:10002/models',
1392+
},
1393+
{
1394+
provider: 'gemini',
1395+
port: 10003,
1396+
base_url: 'http://api-proxy:10003',
1397+
configured: !!GEMINI_API_KEY,
1398+
models: cachedModels.gemini || null,
1399+
models_url: 'http://api-proxy:10003/v1beta/models',
1400+
},
1401+
{
1402+
provider: 'opencode',
1403+
port: 10004,
1404+
base_url: 'http://api-proxy:10004',
1405+
configured: opencodeConfigured,
1406+
// OpenCode routes to one of the above providers; query them directly for models
1407+
models: null,
1408+
models_url: null,
1409+
},
1410+
],
1411+
models_fetch_complete: modelFetchComplete,
1412+
};
1413+
}
1414+
11481415
function healthResponse() {
11491416
return {
11501417
status: 'healthy',
@@ -1166,7 +1433,7 @@ function healthResponse() {
11661433
}
11671434

11681435
/**
1169-
* Handle management endpoints on port 10000 (/health, /metrics).
1436+
* Handle management endpoints on port 10000 (/health, /metrics, /reflect).
11701437
* Returns true if the request was handled, false otherwise.
11711438
*/
11721439
function handleManagementEndpoint(req, res) {
@@ -1180,6 +1447,11 @@ function handleManagementEndpoint(req, res) {
11801447
res.end(JSON.stringify(metrics.getMetrics()));
11811448
return true;
11821449
}
1450+
if (req.method === 'GET' && req.url === '/reflect') {
1451+
res.writeHead(200, { 'Content-Type': 'application/json' });
1452+
res.end(JSON.stringify(reflectEndpoints()));
1453+
return true;
1454+
}
11831455
return false;
11841456
}
11851457

@@ -1205,6 +1477,10 @@ if (require.main === module) {
12051477
logRequest('error', 'key_validation_error', { message: 'Unexpected error during key validation', error: String(err) });
12061478
keyValidationComplete = true;
12071479
});
1480+
fetchStartupModels().catch((err) => {
1481+
logRequest('error', 'model_fetch_error', { message: 'Unexpected error fetching startup models', error: String(err) });
1482+
modelFetchComplete = true;
1483+
});
12081484
}
12091485
}
12101486

@@ -1506,4 +1782,4 @@ if (require.main === module) {
15061782
}
15071783

15081784
// Export for testing
1509-
module.exports = { normalizeApiTarget, deriveCopilotApiTarget, deriveGitHubApiTarget, deriveGitHubApiBasePath, normalizeBasePath, buildUpstreamPath, proxyWebSocket, resolveCopilotAuthToken, resolveOpenCodeRoute, shouldStripHeader, stripGeminiKeyParam, validateApiKeys, probeProvider, httpProbe, keyValidationResults, resetKeyValidationState };
1785+
module.exports = { normalizeApiTarget, deriveCopilotApiTarget, deriveGitHubApiTarget, deriveGitHubApiBasePath, normalizeBasePath, buildUpstreamPath, proxyWebSocket, resolveCopilotAuthToken, resolveOpenCodeRoute, shouldStripHeader, stripGeminiKeyParam, validateApiKeys, probeProvider, httpProbe, keyValidationResults, resetKeyValidationState, fetchJson, extractModelIds, fetchStartupModels, reflectEndpoints, cachedModels, resetModelCacheState };

0 commit comments

Comments
 (0)