Revise to use TS RunResult; Pass in API key

Author: baonguyenNava

scripts/eval_skills/agent.ts

@@ -1,4 +1,4 @@
-import { Agent } from "@cursor/sdk";
+import { Agent, type RunResult } from "@cursor/sdk";
 import { validateAgentResult } from "./schema.js";
 import type { AgentResult } from "./schema.js";
 
@@ -24,6 +24,11 @@ function substitute(template: string, vars: Record<string, string>): string {
 }
 
 export async function runAgent(input: RunAgentInput): Promise<RunAgentOutput> {
+  const apiKey = process.env.CURSOR_API_KEY;
+  if (!apiKey) {
+    return { ok: false, error: "CURSOR_API_KEY env var is not set" };
+  }
+
   const prompt = substitute(input.promptTemplate, {
     SKILL_PATH: input.skillPath,
     SKILL_CONTENT: input.skillContent,
@@ -32,22 +37,31 @@ export async function runAgent(input: RunAgentInput): Promise<RunAgentOutput> {
     RUBRIC: input.rubric,
   });
 
-  let raw: string;
+  let res: RunResult;
   try {
-    const res = await Agent.prompt(prompt, {
+    res = await Agent.prompt(prompt, {
+      apiKey,
+      model: { id: "composer-2" },
       local: { cwd: process.cwd() },
     });
-    if (!res.result) {
-      return { ok: false, error: "agent returned no result" };
-    }
-    raw = res.result;
   } catch (err) {
     return {
       ok: false,
       error: err instanceof Error ? err.message : String(err),
     };
   }
 
+  if (res.status !== "finished") {
+    return {
+      ok: false,
+      error: `agent run ended with status "${res.status}"`,
+    };
+  }
+  if (!res.result) {
+    return { ok: false, error: "agent returned no result" };
+  }
+  const raw = res.result;
+
   let parsed: unknown;
   try {
     parsed = JSON.parse(raw);

tests/eval_skills/agent.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 vi.mock("@cursor/sdk", () => ({
   Agent: {
@@ -14,6 +14,14 @@ vi.mock("@cursor/sdk", () => ({
 import { runAgent } from "../../scripts/eval_skills/agent.js";
 
 describe("runAgent", () => {
+  beforeEach(() => {
+    vi.stubEnv("CURSOR_API_KEY", "test-key");
+  });
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+  });
+
   it("returns a validated AgentResult on a successful call", async () => {
     const r = await runAgent({
       skillPath: "skills/x/SKILL.md",
@@ -59,4 +67,54 @@ describe("runAgent", () => {
     expect(r.ok).toBe(false);
     if (!r.ok) expect(r.error).toContain("api down");
   });
+
+  it("returns ok:false when the run was cancelled", async () => {
+    const sdk = await import("@cursor/sdk");
+    vi.mocked(sdk.Agent.prompt).mockResolvedValueOnce({
+      id: "run-3",
+      status: "cancelled",
+    });
+    const r = await runAgent({
+      skillPath: "skills/x/SKILL.md",
+      skillContent: "x",
+      siblingIndexJson: "[]",
+      promptTemplate: "x",
+      repoRulesExcerpt: "",
+      rubric: "",
+    });
+    expect(r.ok).toBe(false);
+    if (!r.ok) expect(r.error).toContain("cancelled");
+  });
+
+  it("returns ok:false when the run errored", async () => {
+    const sdk = await import("@cursor/sdk");
+    vi.mocked(sdk.Agent.prompt).mockResolvedValueOnce({
+      id: "run-4",
+      status: "error",
+    });
+    const r = await runAgent({
+      skillPath: "skills/x/SKILL.md",
+      skillContent: "x",
+      siblingIndexJson: "[]",
+      promptTemplate: "x",
+      repoRulesExcerpt: "",
+      rubric: "",
+    });
+    expect(r.ok).toBe(false);
+    if (!r.ok) expect(r.error).toContain("error");
+  });
+
+  it("returns ok:false when CURSOR_API_KEY is not set", async () => {
+    vi.stubEnv("CURSOR_API_KEY", "");
+    const r = await runAgent({
+      skillPath: "skills/x/SKILL.md",
+      skillContent: "x",
+      siblingIndexJson: "[]",
+      promptTemplate: "x",
+      repoRulesExcerpt: "",
+      rubric: "",
+    });
+    expect(r.ok).toBe(false);
+    if (!r.ok) expect(r.error).toContain("CURSOR_API_KEY");
+  });
 });