diff --git a/index.ts b/index.ts index a0da40f..f5bff3c 100644 --- a/index.ts +++ b/index.ts @@ -25,24 +25,32 @@ import { } from "./lib/output-test-runner.ts"; import { resultWriteTool, testComponentTool } from "./lib/tools/index.ts"; import { - buildPricingMap, - lookupPricingFromMap, getModelPricingDisplay, formatCost, + formatMTokCost, + buildPricingMap, + lookupPricingFromMap, formatFullPricingDisplay, } from "./lib/pricing.ts"; -import type { LanguageModel } from "ai"; import { - intro, - multiselect, - isCancel, - cancel, - text, - select, - confirm, - note, -} from "@clack/prompts"; -import { gateway } from "ai"; + gateway, + getGatewayModelsAndPricing, + selectModelsFromGateway, + type PricingMap, + type PricingLookup, + type PricingResult, +} from "./lib/providers/ai-gateway.ts"; +import { + configureLMStudio, + selectModelsFromLMStudio, + getLMStudioModel, + isLMStudioModel, + type LMStudioConfig, +} from "./lib/providers/lmstudio.ts"; +import type { LanguageModel } from "ai"; +import { intro, isCancel, cancel, select, confirm, text, multiselect, note } from "@clack/prompts"; + +type ProviderType = "gateway" | "lmstudio"; const SETTINGS_FILE = ".ai-settings.json"; @@ -52,6 +60,7 @@ interface SavedSettings { mcpServerUrl?: string; testingTool: boolean; pricingEnabled: boolean; + provider?: ProviderType; } function loadSettings(): SavedSettings | null { @@ -60,8 +69,8 @@ function loadSettings(): SavedSettings | null { const content = readFileSync(SETTINGS_FILE, "utf-8"); return JSON.parse(content) as SavedSettings; } - } catch (error) { - console.warn("⚠️ Could not load saved settings, using defaults"); + } catch { + console.warn("Could not load saved settings, using defaults"); } return null; } @@ -69,83 +78,45 @@ function loadSettings(): SavedSettings | null { function saveSettings(settings: SavedSettings): void { try { writeFileSync(SETTINGS_FILE, JSON.stringify(settings, null, 2)); - } catch (error) { - console.warn("⚠️ Could not save settings"); + } catch { + console.warn("Could not save settings"); } } -async function validateAndConfirmPricing( - models: string[], - pricingMap: ReturnType, - savedPricingEnabled?: boolean, -) { - const lookups = new Map>(); - - for (const modelId of models) { - const lookup = lookupPricingFromMap(modelId, pricingMap); - lookups.set(modelId, lookup); - } - - const modelsWithPricing = models.filter((m) => lookups.get(m) !== null); - const modelsWithoutPricing = models.filter((m) => lookups.get(m) === null); - - if (modelsWithoutPricing.length === 0) { - const pricingLines = models.map((modelId) => { - const lookup = lookups.get(modelId)!; - const display = getModelPricingDisplay(lookup.pricing); - return `${modelId}\n → ${formatFullPricingDisplay(display)}`; - }); - - note(pricingLines.join("\n\n"), "💰 Pricing Found"); - - const usePricing = await confirm({ - message: "Enable cost calculation?", - initialValue: savedPricingEnabled ?? true, - }); - - if (isCancel(usePricing)) { - cancel("Operation cancelled."); - process.exit(0); - } - - return { enabled: usePricing, lookups }; - } else { - const lines: string[] = []; - - if (modelsWithoutPricing.length > 0) { - lines.push("No pricing found for:"); - for (const modelId of modelsWithoutPricing) { - lines.push(` ✗ ${modelId}`); - } - } - - if (modelsWithPricing.length > 0) { - lines.push(""); - lines.push("Pricing available for:"); - for (const modelId of modelsWithPricing) { - const lookup = lookups.get(modelId)!; - const display = getModelPricingDisplay(lookup.pricing); - lines.push(` ✓ ${modelId} (${formatFullPricingDisplay(display)})`); - } - } - - lines.push(""); - lines.push("Cost calculation will be disabled."); - - note(lines.join("\n"), "⚠️ Pricing Incomplete"); +interface ProviderConfig { + type: ProviderType; + lmstudio?: LMStudioConfig; +} - const proceed = await confirm({ - message: "Continue without pricing?", - initialValue: true, - }); +async function selectProvider(savedProvider?: ProviderType): Promise { + const provider = await select({ + message: "Select model provider", + options: [ + { + value: "gateway", + label: "Vercel AI Gateway", + hint: "Cloud-hosted models via Vercel", + }, + { + value: "lmstudio", + label: "LM Studio", + hint: "Local models via LM Studio", + }, + ], + initialValue: savedProvider ?? "gateway", + }); - if (isCancel(proceed) || !proceed) { - cancel("Operation cancelled."); - process.exit(0); - } + if (isCancel(provider)) { + cancel("Operation cancelled."); + process.exit(0); + } - return { enabled: false, lookups }; + if (provider === "lmstudio") { + const lmstudioConfig = await configureLMStudio(); + return { type: "lmstudio", lmstudio: lmstudioConfig }; } + + return { type: "gateway" }; } async function selectOptions() { @@ -153,59 +124,32 @@ async function selectOptions() { const savedSettings = loadSettings(); if (savedSettings) { - note("Loaded previous settings as defaults", "📋 Saved Settings"); + note("Loaded previous settings as defaults", "Saved Settings"); } - const availableModels = await gateway.getAvailableModels(); + const providerConfig = await selectProvider(savedSettings?.provider); - const pricingMap = buildPricingMap(availableModels.models); - - const modelOptions = [{ value: "custom", label: "Custom" }].concat( - availableModels.models.reduce>( - (arr, model) => { - if (model.modelType === "language") { - arr.push({ value: model.id, label: model.name }); - } - return arr; - }, - [], - ), - ); - - const savedModelValues = savedSettings?.models ?? []; - - const models = await multiselect({ - message: "Select model(s) to benchmark", - options: modelOptions, - initialValues: savedModelValues.filter((m) => - modelOptions.some((opt) => opt.value === m), - ), - }); - - if (isCancel(models)) { - cancel("Operation cancelled."); - process.exit(0); - } + let pricingMap: PricingMap; + let selectedModels: string[]; + let pricing: PricingResult; - if (models.includes("custom")) { - const customModel = await text({ - message: "Enter custom model id", - }); - if (isCancel(customModel)) { - cancel("Operation cancelled."); - process.exit(0); - } - models.push(customModel); + if (providerConfig.type === "gateway") { + const gatewayData = await getGatewayModelsAndPricing(); + pricingMap = gatewayData.pricingMap; + const result = await selectModelsFromGateway(pricingMap, savedSettings?.pricingEnabled); + selectedModels = result.selectedModels; + pricing = result.pricing; + } else { + pricingMap = buildPricingMap([]); + selectedModels = await selectModelsFromLMStudio( + providerConfig.lmstudio!.baseURL, + ); + pricing = { + enabled: false, + lookups: new Map(), + }; } - const selectedModels = models.filter((model) => model !== "custom"); - - const pricing = await validateAndConfirmPricing( - selectedModels, - pricingMap, - savedSettings?.pricingEnabled, - ); - const savedMcpIntegration = savedSettings?.mcpIntegration ?? "none"; const mcpIntegration = await select({ @@ -290,6 +234,7 @@ async function selectOptions() { mcp, testingTool, pricing, + providerConfig, }; } @@ -303,6 +248,17 @@ function parseCommandString(commandString: string): { return { command, args }; } +function getModelForId( + modelId: string, + providerConfig: ProviderConfig, +): LanguageModel { + if (isLMStudioModel(modelId)) { + return getLMStudioModel(modelId, providerConfig.lmstudio?.baseURL); + } + + return gateway.languageModel(modelId); +} + async function runSingleTest( test: TestDefinition, model: LanguageModel, @@ -484,7 +440,8 @@ async function runSingleTest( } async function main() { - const { models, mcp, testingTool, pricing } = await selectOptions(); + const { models, mcp, testingTool, pricing, providerConfig } = + await selectOptions(); const mcpServerUrl = mcp; const mcpEnabled = !!mcp; @@ -498,6 +455,13 @@ async function main() { console.log("║ SvelteBench 2.0 - Multi-Test ║"); console.log("╚════════════════════════════════════════════════════╝"); + console.log( + `\n🔌 Provider: ${providerConfig.type === "gateway" ? "Vercel AI Gateway" : "LM Studio"}`, + ); + if (providerConfig.type === "lmstudio" && providerConfig.lmstudio) { + console.log(` URL: ${providerConfig.lmstudio.baseURL}`); + } + console.log("\n📋 Models:"); for (const modelId of models) { const lookup = pricing.lookups.get(modelId); @@ -507,6 +471,9 @@ async function main() { console.log(` 💰 ${formatFullPricingDisplay(display)}`); } else { console.log(` ${modelId}`); + if (isLMStudioModel(modelId)) { + console.log(` 🖥️ Local model (free)`); + } } } @@ -575,7 +542,7 @@ async function main() { console.log(`💰 Pricing: ${formatFullPricingDisplay(display)}`); } - const model = gateway.languageModel(modelId); + const model = getModelForId(modelId, providerConfig); const testResults: SingleTestResult[] = []; const startTime = Date.now(); @@ -726,10 +693,17 @@ async function main() { mcpTransportType: mcpEnabled ? mcpTransportType : null, timestamp: new Date().toISOString(), model: modelId, + provider: providerConfig.type, pricingKey: pricingLookup?.matchedKey ?? null, pricing: pricingInfo, totalCost, cacheSimulation, + lmstudio: + providerConfig.type === "lmstudio" && providerConfig.lmstudio + ? { + baseURL: providerConfig.lmstudio.baseURL, + } + : null, unitTestTotals, }, }; diff --git a/lib/providers/ai-gateway.ts b/lib/providers/ai-gateway.ts new file mode 100644 index 0000000..cbb3b7f --- /dev/null +++ b/lib/providers/ai-gateway.ts @@ -0,0 +1,159 @@ +import { gateway } from "ai"; +import { multiselect, isCancel, cancel, text, confirm, note } from "@clack/prompts"; +import { + buildPricingMap, + lookupPricingFromMap, + getModelPricingDisplay, + formatMTokCost, +} from "../pricing.ts"; + +export { gateway }; + +export type PricingMap = ReturnType; +export type PricingLookup = ReturnType; + +export interface PricingResult { + enabled: boolean; + lookups: Map; +} + +export async function getGatewayModelsAndPricing() { + const availableModels = await gateway.getAvailableModels(); + const pricingMap = buildPricingMap(availableModels.models); + return { models: availableModels.models, pricingMap }; +} + +export async function validateAndConfirmPricing( + models: string[], + pricingMap: PricingMap, + savedPricingEnabled?: boolean, +): Promise { + const lookups = new Map(); + + for (const modelId of models) { + const lookup = lookupPricingFromMap(modelId, pricingMap); + lookups.set(modelId, lookup); + } + + const modelsWithPricing = models.filter((m) => lookups.get(m) !== null); + const modelsWithoutPricing = models.filter((m) => lookups.get(m) === null); + + if (modelsWithoutPricing.length === 0) { + const pricingLines = models.map((modelId) => { + const lookup = lookups.get(modelId)!; + const display = getModelPricingDisplay(lookup.pricing); + const cacheReadText = + display.cacheReadCostPerMTok !== undefined + ? `, ${formatMTokCost(display.cacheReadCostPerMTok)}/MTok cache read` + : ""; + const cacheWriteText = + display.cacheCreationCostPerMTok !== undefined + ? `, ${formatMTokCost(display.cacheCreationCostPerMTok)}/MTok cache write` + : ""; + return `${modelId}\n → ${formatMTokCost(display.inputCostPerMTok)}/MTok in, ${formatMTokCost(display.outputCostPerMTok)}/MTok out${cacheReadText}${cacheWriteText}`; + }); + + note(pricingLines.join("\n\n"), "💰 Pricing Found"); + + const usePricing = await confirm({ + message: "Enable cost calculation?", + initialValue: savedPricingEnabled ?? true, + }); + + if (isCancel(usePricing)) { + cancel("Operation cancelled."); + process.exit(0); + } + + return { enabled: usePricing, lookups }; + } else { + const lines: string[] = []; + + if (modelsWithoutPricing.length > 0) { + lines.push("No pricing found for:"); + for (const modelId of modelsWithoutPricing) { + lines.push(` ✗ ${modelId}`); + } + } + + if (modelsWithPricing.length > 0) { + lines.push(""); + lines.push("Pricing available for:"); + for (const modelId of modelsWithPricing) { + const lookup = lookups.get(modelId)!; + const display = getModelPricingDisplay(lookup.pricing); + const cacheReadText = + display.cacheReadCostPerMTok !== undefined + ? `, ${formatMTokCost(display.cacheReadCostPerMTok)}/MTok cache read` + : ""; + const cacheWriteText = + display.cacheCreationCostPerMTok !== undefined + ? `, ${formatMTokCost(display.cacheCreationCostPerMTok)}/MTok cache write` + : ""; + lines.push( + ` ✓ ${modelId} (${formatMTokCost(display.inputCostPerMTok)}/MTok in, ${formatMTokCost(display.outputCostPerMTok)}/MTok out${cacheReadText}${cacheWriteText})`, + ); + } + } + + lines.push(""); + lines.push("Cost calculation will be disabled."); + + note(lines.join("\n"), "⚠️ Pricing Incomplete"); + + const proceed = await confirm({ + message: "Continue without pricing?", + initialValue: true, + }); + + if (isCancel(proceed) || !proceed) { + cancel("Operation cancelled."); + process.exit(0); + } + + return { enabled: false, lookups }; + } +} + +export async function selectModelsFromGateway( + pricingMap: PricingMap, + savedPricingEnabled?: boolean, +) { + const availableModels = await gateway.getAvailableModels(); + + const models = await multiselect({ + message: "Select model(s) to benchmark", + options: [{ value: "custom", label: "Custom" }].concat( + availableModels.models.reduce>( + (arr, model) => { + if (model.modelType === "language") { + arr.push({ value: model.id, label: model.name }); + } + return arr; + }, + [], + ), + ), + }); + + if (isCancel(models)) { + cancel("Operation cancelled."); + process.exit(0); + } + + if (models.includes("custom")) { + const customModel = await text({ + message: "Enter custom model id", + }); + if (isCancel(customModel)) { + cancel("Operation cancelled."); + process.exit(0); + } + models.push(customModel); + } + + const selectedModels = models.filter((model) => model !== "custom"); + const pricing = await validateAndConfirmPricing(selectedModels, pricingMap, savedPricingEnabled); + + return { selectedModels, pricing }; +} diff --git a/lib/providers/lmstudio.ts b/lib/providers/lmstudio.ts new file mode 100644 index 0000000..f763d5b --- /dev/null +++ b/lib/providers/lmstudio.ts @@ -0,0 +1,154 @@ +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; +import { + multiselect, + isCancel, + cancel, + confirm, + text, + spinner, + note, +} from "@clack/prompts"; +import type { LanguageModel } from "ai"; + +export function createLMStudioProvider( + baseURL: string = "http://localhost:1234/v1", +) { + return createOpenAICompatible({ + name: "lmstudio", + baseURL, + }); +} + +export const lmstudio = createLMStudioProvider(); + +export interface LMStudioModel { + id: string; + object: string; + owned_by: string; +} + +interface LMStudioModelsResponse { + object: string; + data: LMStudioModel[]; +} + +export interface LMStudioConfig { + baseURL: string; +} + +export async function fetchLMStudioModels( + baseURL: string = "http://localhost:1234/v1", +): Promise { + try { + const response = await fetch(`${baseURL}/models`, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + console.error( + `Failed to fetch LM Studio models: ${response.status} ${response.statusText}`, + ); + return null; + } + + const data = (await response.json()) as LMStudioModelsResponse; + return data.data || []; + } catch { + return null; + } +} + +export async function configureLMStudio(): Promise { + const customUrl = await confirm({ + message: "Use custom LM Studio URL? (default: http://localhost:1234/v1)", + initialValue: false, + }); + + if (isCancel(customUrl)) { + cancel("Operation cancelled."); + process.exit(0); + } + + let baseURL = "http://localhost:1234/v1"; + + if (customUrl) { + const urlInput = await text({ + message: "Enter LM Studio server URL", + placeholder: "http://localhost:1234/v1", + }); + + if (isCancel(urlInput)) { + cancel("Operation cancelled."); + process.exit(0); + } + + baseURL = urlInput || "http://localhost:1234/v1"; + } + + return { baseURL }; +} + +export async function selectModelsFromLMStudio( + baseURL: string, +): Promise { + const s = spinner(); + s.start("Connecting to LM Studio..."); + + const lmstudioModels = await fetchLMStudioModels(baseURL); + + if (lmstudioModels === null) { + s.stop("Failed to connect to LM Studio"); + note( + `Could not connect to LM Studio at ${baseURL}\n\nMake sure:\n1. LM Studio is running\n2. A model is loaded\n3. The local server is started (Local Server tab → Start Server)`, + "❌ Connection Failed", + ); + cancel("Cannot proceed without LM Studio connection."); + process.exit(1); + } + + if (lmstudioModels.length === 0) { + s.stop("No models found"); + note( + `LM Studio is running but no models are loaded.\n\nPlease load a model in LM Studio and try again.`, + "⚠️ No Models Available", + ); + cancel("Cannot proceed without loaded models."); + process.exit(1); + } + + s.stop(`Found ${lmstudioModels.length} model(s)`); + + const models = await multiselect({ + message: "Select model(s) to benchmark", + options: lmstudioModels.map((model) => ({ + value: model.id, + label: model.id, + hint: model.owned_by !== "unknown" ? `by ${model.owned_by}` : undefined, + })), + }); + + if (isCancel(models)) { + cancel("Operation cancelled."); + process.exit(0); + } + + return models.map((m) => `lmstudio/${m}`); +} + +export function getLMStudioModel( + modelId: string, + baseURL?: string, +): LanguageModel { + const actualModelId = modelId.startsWith("lmstudio/") + ? modelId.replace("lmstudio/", "") + : modelId; + const provider = createLMStudioProvider(baseURL); + return provider(actualModelId); +} + +export function isLMStudioModel(modelId: string): boolean { + return modelId.startsWith("lmstudio/"); +} diff --git a/lib/report.ts b/lib/report.ts index 39b4651..22653eb 100644 --- a/lib/report.ts +++ b/lib/report.ts @@ -93,16 +93,22 @@ export interface UnitTestTotals { score: number; } +interface LMStudioMetadata { + baseURL: string; +} + interface Metadata { mcpEnabled: boolean; mcpServerUrl: string | null; mcpTransportType?: string | null; timestamp: string; model: string; + provider?: "gateway" | "lmstudio"; pricingKey?: string | null; pricing?: PricingInfo | null; totalCost?: TotalCostInfo | null; cacheSimulation?: ReturnType | null; + lmstudio?: LMStudioMetadata | null; unitTestTotals: UnitTestTotals; }