Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions __tests__/api/mcp.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import { describe, it, expect } from "vitest";
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
import { createMcpServer } from "@/app/api/mcp/server";

type TextContent = { type: "text"; text: string };

async function createConnectedClient() {
const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
const server = createMcpServer();
await server.connect(serverTransport);
const client = new Client({ name: "test-client", version: "1.0.0" });
await client.connect(clientTransport);
return client;
}

function firstTextContent(result: Awaited<ReturnType<Client["callTool"]>>): TextContent {
const content = result.content as TextContent[];
return content[0];
}

describe("MCP server", () => {
it("lists all expected tools", async () => {
const client = await createConnectedClient();
const { tools } = await client.listTools();
const names = tools.map((t) => t.name);

expect(names).toContain("ping");
expect(names).toContain("roast_stack");
expect(names).toContain("challenge_stack");
expect(names).toContain("get_stack_questions");
expect(names).toContain("recommend_stack");
});

it("ping returns pong", async () => {
const client = await createConnectedClient();
const result = await client.callTool({ name: "ping", arguments: {} });
const content = firstTextContent(result);
expect(content.type).toBe("text");
expect(JSON.parse(content.text)).toEqual({ pong: true });
});

it("get_stack_questions returns all 4 questions with options", async () => {
const client = await createConnectedClient();
const result = await client.callTool({ name: "get_stack_questions", arguments: {} });
const { questions } = JSON.parse(firstTextContent(result).text);

expect(questions).toHaveLength(4);
const ids = questions.map((q: { id: string }) => q.id);
expect(ids).toEqual(["what", "who", "priority", "budget"]);

for (const q of questions) {
expect(q.options.length).toBeGreaterThan(0);
expect(typeof q.text).toBe("string");
expect(typeof q.hint).toBe("string");
}
});

it("roast_stack returns no_valid_tools when no names match catalog", async () => {
const client = await createConnectedClient();
const result = await client.callTool({
name: "roast_stack",
arguments: { tools: ["__nonexistent_tool__"] },
});
const body = JSON.parse(firstTextContent(result).text);
expect(body.error).toBe("no_valid_tools");
expect(body.skipped).toContain("__nonexistent_tool__");
});

it("challenge_stack returns no_valid_tools when no names match catalog", async () => {
const client = await createConnectedClient();
const result = await client.callTool({
name: "challenge_stack",
arguments: { tools: ["__nonexistent__"] },
});
const body = JSON.parse(firstTextContent(result).text);
expect(body.error).toBe("no_valid_tools");
expect(body.skipped).toContain("__nonexistent__");
});
});
92 changes: 92 additions & 0 deletions __tests__/lib/resolveTools.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import { describe, it, expect } from "vitest";
import { resolveTools } from "@/lib/ai/resolveTools";
import type { Tool } from "@/lib/types";

function makeTool(overrides: Partial<Tool> & { id: string; name: string }): Tool {
return {
category: "coding-assistants",
tagline: "",
description: "",
type: "commercial",
pricing: { free_tier: false, plans: [] },
github_stars: null,
slot: "coding-assistant",
website_url: null,
github_url: null,
use_context: "dev-productivity",
...overrides,
};
}

const CATALOG: Tool[] = [
makeTool({ id: "cursor", name: "Cursor" }),
makeTool({
id: "langgraph",
name: "LangGraph",
aliases: { npm: ["@langchain/langgraph"], pip: ["langgraph"], env_vars: [], config_files: [] },
}),
makeTool({
id: "openai",
name: "OpenAI",
aliases: { npm: [], pip: [], env_vars: ["OPENAI_API_KEY"], config_files: [] },
}),
makeTool({ id: "supabase", name: "Supabase" }),
];

describe("resolveTools", () => {
it("matches by exact id", () => {
const { resolved, skipped } = resolveTools(["cursor"], CATALOG);
expect(resolved).toHaveLength(1);
expect(resolved[0].tool.id).toBe("cursor");
expect(skipped).toHaveLength(0);
});

it("matches by case-insensitive name", () => {
const { resolved } = resolveTools(["Cursor", "SUPABASE", "langgraph"], CATALOG);
expect(resolved.map((r) => r.tool.id)).toEqual(["cursor", "supabase", "langgraph"]);
});

it("matches hyphenated id when input has the same normalized form", () => {
const catalog = [...CATALOG, makeTool({ id: "claude-code", name: "Claude Code" })];
// "Claude_Code" normalizes to "claude-code" which matches the id
const { resolved } = resolveTools(["Claude_Code"], catalog);
expect(resolved[0].tool.id).toBe("claude-code");
});

it("matches by npm alias", () => {
const { resolved } = resolveTools(["@langchain/langgraph"], CATALOG);
expect(resolved[0].tool.id).toBe("langgraph");
});

it("matches by pip alias", () => {
const { resolved } = resolveTools(["langgraph"], CATALOG);
expect(resolved[0].tool.id).toBe("langgraph");
});

it("matches by env_var alias", () => {
const { resolved } = resolveTools(["OPENAI_API_KEY"], CATALOG);
expect(resolved[0].tool.id).toBe("openai");
});

it("puts unmatched names in skipped", () => {
const { resolved, skipped } = resolveTools(["Cursor", "UnknownTool"], CATALOG);
expect(resolved).toHaveLength(1);
expect(skipped).toEqual(["UnknownTool"]);
});

it("deduplicates when the same tool is matched twice", () => {
const { resolved } = resolveTools(["cursor", "Cursor"], CATALOG);
expect(resolved).toHaveLength(1);
});

it("returns empty resolved and all skipped when nothing matches", () => {
const { resolved, skipped } = resolveTools(["__x__", "__y__"], CATALOG);
expect(resolved).toHaveLength(0);
expect(skipped).toEqual(["__x__", "__y__"]);
});

it("preserves the original input string in resolved", () => {
const { resolved } = resolveTools(["Cursor"], CATALOG);
expect(resolved[0].input).toBe("Cursor");
});
});
112 changes: 12 additions & 100 deletions app/api/challenge/route.ts
Original file line number Diff line number Diff line change
@@ -1,98 +1,28 @@
import { GoogleGenerativeAI } from "@google/generative-ai";
import { NextResponse } from "next/server";
import { generateChallenge, type ChallengeInput } from "@/lib/ai/challenge";

export const dynamic = "force-dynamic";

export interface ChallengeRequest {
filledSlots: { slotName: string; toolName: string }[];
missingRequired: string[];
tier: string;
fitnessScore: number;
archetype: string;
}

export interface ChallengeItem {
tool: string;
challenge: string;
recommendation: string;
}

export interface ChallengeResponse {
challenges: ChallengeItem[];
}

const SYSTEM_INSTRUCTION = `You are an adversarial AI stack reviewer. Your job is to argue against specific tool choices — not mock them, but challenge them rigorously using reasoning about scale, lock-in, operational cost, debugging overhead, and architectural tradeoffs.

Rules:
- Output ONLY a JSON object: {"challenges": [{...}, ...]}
- 3 to 5 challenges. Target the most questionable tool choices first.
- Each challenge must have exactly three fields:
- "tool": the exact tool name from the input
- "challenge": one specific, grounded argument against this choice (1–2 sentences, max 160 chars). Name the specific risk — not a generic concern.
- "recommendation": one actionable next step or condition that would change the analysis (1 sentence, max 120 chars).
- Only target tools from the filled slots list. Do not invent tools.
- If a tool choice is genuinely defensible for this stack, challenge a different one instead.
- No flattery, no padding, no markdown outside the JSON object.`;

export async function POST(request: Request) {
const apiKey = process.env.GOOGLE_AI_API_KEY;
if (!apiKey) {
return NextResponse.json({ error: "GOOGLE_AI_API_KEY not configured" }, { status: 503 });
}

let body: ChallengeRequest;
let body: ChallengeInput;
try {
body = await request.json();
} catch {
return NextResponse.json({ error: "Invalid JSON" }, { status: 400 });
}

const { filledSlots, missingRequired, tier, fitnessScore, archetype } = body;

if (!filledSlots || filledSlots.length === 0) {
if (!body.filledSlots || body.filledSlots.length === 0) {
return NextResponse.json({ error: "No tools to challenge" }, { status: 400 });
}

const userPrompt = buildPrompt({ filledSlots, missingRequired, tier, fitnessScore, archetype });

try {
const genAI = new GoogleGenerativeAI(apiKey);
const model = genAI.getGenerativeModel({
model: "gemini-2.5-flash",
systemInstruction: SYSTEM_INSTRUCTION,
});

const result = await model.generateContent(userPrompt);
const text = result.response.text().trim();

let challenges: ChallengeItem[] = [];
try {
const clean = text.replace(/^```(?:json)?\n?/, "").replace(/\n?```$/, "");
const parsed = JSON.parse(clean);
if (Array.isArray(parsed.challenges)) {
challenges = parsed.challenges
.filter(
(c: unknown) =>
typeof c === "object" &&
c !== null &&
"tool" in c &&
"challenge" in c &&
"recommendation" in c
)
.slice(0, 5) as ChallengeItem[];
}
} catch {
// If JSON parse fails, return empty — no fallback for structured data
}

if (challenges.length === 0) {
return NextResponse.json({ error: "Failed to generate challenges" }, { status: 500 });
}

return NextResponse.json({ challenges } satisfies ChallengeResponse);
const result = await generateChallenge(body);
return NextResponse.json(result);
} catch (err) {
console.error("[challenge] Gemini API error:", err);
const msg = err instanceof Error ? err.message : String(err);
if (msg === "GOOGLE_AI_API_KEY not configured") {
return NextResponse.json({ error: msg }, { status: 503 });
}
if (
msg.includes("429") ||
msg.toLowerCase().includes("quota") ||
Expand All @@ -103,27 +33,9 @@ export async function POST(request: Request) {
{ status: 429 }
);
}
return NextResponse.json({ error: "Gemini API error" }, { status: 500 });
return NextResponse.json(
{ error: "Failed to generate challenges. Try again." },
{ status: 500 }
);
}
}

function buildPrompt(data: ChallengeRequest): string {
const lines: string[] = [];

lines.push(`Stack archetype: ${data.archetype}`);
lines.push(`Stack tier: ${data.tier} (${data.fitnessScore}/100)`);
lines.push(`\nFilled slots (slot → tool chosen):`);
for (const s of data.filledSlots) {
lines.push(` ${s.slotName}: ${s.toolName}`);
}

if (data.missingRequired.length > 0) {
lines.push(`\nMissing required layers: ${data.missingRequired.join(", ")}`);
}

lines.push(
`\nChallenge the most questionable tool choices above. Be specific about why each choice could be wrong for this archetype and tier.`
);

return lines.join("\n");
}
32 changes: 32 additions & 0 deletions app/api/mcp/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
import { createMcpServer } from "./server";

export const dynamic = "force-dynamic";

function makeTransport() {
return new WebStandardStreamableHTTPServerTransport({
sessionIdGenerator: undefined, // stateless — no session management
enableJsonResponse: true,
});
}

export async function POST(req: Request) {
const transport = makeTransport();
const server = createMcpServer();
await server.connect(transport);
return transport.handleRequest(req);
}

export async function GET(req: Request) {
const transport = makeTransport();
const server = createMcpServer();
await server.connect(transport);
return transport.handleRequest(req);
}

export async function DELETE(req: Request) {
const transport = makeTransport();
const server = createMcpServer();
await server.connect(transport);
return transport.handleRequest(req);
}
20 changes: 20 additions & 0 deletions app/api/mcp/server.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import { registerPing } from "./tools/ping";
import { registerRoastStack } from "./tools/roast";
import { registerChallengeStack } from "./tools/challenge";
import { registerGetStackQuestions, registerRecommendStack } from "./tools/recommend";

export function createMcpServer(): McpServer {
const server = new McpServer({
name: "aichitect",
version: "1.0.0",
});

registerPing(server);
registerRoastStack(server);
registerChallengeStack(server);
registerGetStackQuestions(server);
registerRecommendStack(server);

return server;
}
Loading
Loading