feat(evals): suite-level Default Execution Config editor (#2043)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: chelojimenez

mcpjam-inspector/client/src/components/CiEvalsTab.tsx

@@ -282,6 +282,7 @@ export function CiEvalsTab({
     latestRunBySuiteId,
     evalsNavigationContext: "ci-evals",
     projectServers: ciProjectServers,
+    availableModels,
   });
 
   const suiteAggregate = useMemo(() => {

mcpjam-inspector/client/src/components/EvalsTab.tsx

@@ -150,6 +150,7 @@ export function EvalsTab({
     latestRunBySuiteId,
     projectServers,
     isDirectGuest,
+    availableModels,
   });
   const {
     deletingSuiteId,

mcpjam-inspector/client/src/components/evals/eval-runner.tsx

@@ -52,6 +52,8 @@ interface EvalRunnerProps {
   inline?: boolean;
   onSuccess?: (suiteId?: string) => void;
   preselectedServer?: string;
+  /** Pre-populate model selection from suite.defaultConfig.modelId. User can still change it. */
+  defaultModelId?: string;
 }
 
 type StepKey = (typeof WIZARD_STEPS)[number]["key"];
@@ -151,6 +153,7 @@ export function EvalRunner({
   inline = false,
   onSuccess,
   preselectedServer,
+  defaultModelId,
 }: EvalRunnerProps) {
   const [open, setOpen] = useState(false);
   const [isSubmitting, setIsSubmitting] = useState(false);
@@ -260,6 +263,18 @@ export function EvalRunner({
       return;
     }
 
+    // Suite defaultConfig takes precedence over stored preferences when present
+    if (defaultModelId) {
+      const suiteDefault = availableModels.find(
+        (m) => String(m.id) === defaultModelId,
+      );
+      if (suiteDefault) {
+        setSelectedModels([suiteDefault]);
+        setHasRestoredPreferences(true);
+        return;
+      }
+    }
+
     if (savedPreferences?.modelIds && savedPreferences.modelIds.length > 0) {
       const matches = availableModels.filter((model) =>
         savedPreferences.modelIds.includes(model.id),
@@ -270,7 +285,7 @@ export function EvalRunner({
     }
 
     setHasRestoredPreferences(true);
-  }, [availableModels, savedPreferences, hasRestoredPreferences]);
+  }, [availableModels, savedPreferences, hasRestoredPreferences, defaultModelId]);
 
   useEffect(() => {
     if (typeof window === "undefined") return;

mcpjam-inspector/client/src/components/evals/suite-execution-config-editor.tsx

@@ -0,0 +1,260 @@
+import { useEffect, useState } from "react";
+import { Loader2, Settings2, Trash2 } from "lucide-react";
+import { Button } from "@mcpjam/design-system/button";
+import { Label } from "@mcpjam/design-system/label";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@mcpjam/design-system/select";
+import { Slider } from "@mcpjam/design-system/slider";
+import { Textarea } from "@mcpjam/design-system/textarea";
+import type { EvalSuite } from "./types";
+import type { ModelDefinition } from "@/shared/types";
+
+type SuiteExecutionConfigEditorProps = {
+  suite: Pick<EvalSuite, "_id" | "defaultConfig">;
+  availableModels: ModelDefinition[];
+  onSave: (
+    defaultConfig: NonNullable<EvalSuite["defaultConfig"]>
+  ) => Promise<void>;
+  onClear?: () => Promise<void>;
+};
+
+const DEFAULT_TEMPERATURE = 0.7;
+
+export function SuiteExecutionConfigEditor({
+  suite,
+  availableModels,
+  onSave,
+  onClear,
+}: SuiteExecutionConfigEditorProps) {
+  const [modelId, setModelId] = useState(suite.defaultConfig?.modelId ?? "");
+  const [provider, setProvider] = useState(suite.defaultConfig?.provider ?? "");
+  const [systemPrompt, setSystemPrompt] = useState(
+    suite.defaultConfig?.systemPrompt ?? ""
+  );
+  const [temperature, setTemperature] = useState(
+    suite.defaultConfig?.temperature ?? DEFAULT_TEMPERATURE
+  );
+  const [isSaving, setIsSaving] = useState(false);
+  const [isClearing, setIsClearing] = useState(false);
+
+  // Depend on scalar values, not the object reference: a parent re-render
+  // that produces a fresh `suite.defaultConfig` with identical values would
+  // otherwise stomp in-progress edits. `suite._id` is also a dep so drafts
+  // can't leak across suites when the editor stays mounted (parent doesn't
+  // key it by suite id) and two suites happen to share the same scalars.
+  useEffect(() => {
+    setModelId(suite.defaultConfig?.modelId ?? "");
+    setProvider(suite.defaultConfig?.provider ?? "");
+    setSystemPrompt(suite.defaultConfig?.systemPrompt ?? "");
+    setTemperature(suite.defaultConfig?.temperature ?? DEFAULT_TEMPERATURE);
+  }, [
+    suite._id,
+    suite.defaultConfig?.modelId,
+    suite.defaultConfig?.provider,
+    suite.defaultConfig?.systemPrompt,
+    suite.defaultConfig?.temperature,
+  ]);
+
+  // For suites saved before provider was tracked, fall back to the first
+  // matching model so the Select still renders the saved choice. Computed at
+  // render time so we don't have to add availableModels as an effect dep
+  // (which would risk stomping in-progress edits on parent re-renders).
+  const displayProvider =
+    provider ||
+    (modelId
+      ? availableModels.find((m) => String(m.id) === modelId)?.provider ?? ""
+      : "");
+
+  const savedModelId = suite.defaultConfig?.modelId ?? "";
+  const savedProvider = suite.defaultConfig?.provider ?? "";
+  const savedSystemPrompt = suite.defaultConfig?.systemPrompt ?? "";
+  const savedTemperature =
+    suite.defaultConfig?.temperature ?? DEFAULT_TEMPERATURE;
+
+  const isDirty =
+    modelId !== savedModelId ||
+    provider !== savedProvider ||
+    systemPrompt !== savedSystemPrompt ||
+    temperature !== savedTemperature;
+
+  const handleReset = () => {
+    setModelId(savedModelId);
+    setProvider(savedProvider);
+    setSystemPrompt(savedSystemPrompt);
+    setTemperature(savedTemperature);
+  };
+
+  const handleSave = async () => {
+    if (!modelId) return;
+    setIsSaving(true);
+    try {
+      await onSave({
+        modelId,
+        provider: provider || displayProvider || undefined,
+        systemPrompt,
+        temperature,
+      });
+    } finally {
+      setIsSaving(false);
+    }
+  };
+
+  const handleClear = async () => {
+    if (!onClear) return;
+    setIsClearing(true);
+    try {
+      await onClear();
+    } finally {
+      setIsClearing(false);
+    }
+  };
+
+  return (
+    <section className="space-y-3">
+      <div>
+        <h2 className="text-base font-semibold text-foreground">
+          Default Execution Config
+        </h2>
+        <p className="mt-1 text-xs text-muted-foreground">
+          The model and parameters all iterations in this suite inherit. Per-case{" "}
+          <code className="rounded bg-muted px-1 py-0.5 text-[10px]">
+            advancedConfig
+          </code>{" "}
+          overrides take precedence.
+        </p>
+      </div>
+
+      <div className="space-y-4 rounded-xl border bg-card/60 p-4">
+        {/* Model */}
+        <div>
+          <Label className="text-xs font-medium text-muted-foreground">
+            Model
+          </Label>
+          {/* Encode provider into the Select value so colliding model ids
+              across providers (e.g. native OpenAI gpt-4o vs OpenRouter
+              gpt-4o) are saved with the correct provider. */}
+          <Select
+            value={modelId ? `${displayProvider}:${modelId}` : ""}
+            onValueChange={(value) => {
+              const sep = value.indexOf(":");
+              const nextProvider = sep >= 0 ? value.slice(0, sep) : "";
+              const nextId = sep >= 0 ? value.slice(sep + 1) : value;
+              setProvider(nextProvider);
+              setModelId(nextId);
+            }}
+            disabled={isSaving || isClearing}
+          >
+            <SelectTrigger className="mt-1.5 border-0 bg-muted/50 transition-colors hover:bg-muted">
+              <SelectValue placeholder="Select a model" />
+            </SelectTrigger>
+            <SelectContent>
+              {availableModels.map((model) => {
+                const value = `${model.provider}:${String(model.id)}`;
+                return (
+                  <SelectItem key={value} value={value}>
+                    {model.name}
+                  </SelectItem>
+                );
+              })}
+            </SelectContent>
+          </Select>
+        </div>
+
+        {/* System prompt */}
+        <div>
+          <Label className="text-xs font-medium text-muted-foreground">
+            System prompt
+          </Label>
+          <p className="mb-1.5 mt-0.5 text-[10px] text-muted-foreground">
+            Instructions given to the model at the start of each run.
+          </p>
+          <Textarea
+            value={systemPrompt}
+            onChange={(e) => setSystemPrompt(e.target.value)}
+            placeholder="You are a helpful assistant…"
+            className="min-h-[80px] resize-y border-0 bg-muted/50 text-sm"
+            disabled={isSaving || isClearing}
+          />
+        </div>
+
+        {/* Temperature */}
+        <div>
+          <div className="flex items-center justify-between">
+            <Label className="text-xs font-medium text-muted-foreground">
+              Temperature
+            </Label>
+            <span className="text-xs text-muted-foreground">
+              {temperature.toFixed(2)}
+            </span>
+          </div>
+          <Slider
+            min={0}
+            max={2}
+            step={0.05}
+            value={[temperature]}
+            onValueChange={(values) =>
+              setTemperature(values[0] ?? DEFAULT_TEMPERATURE)
+            }
+            className="mt-3"
+            disabled={isSaving || isClearing}
+          />
+        </div>
+      </div>
+
+      <div className="flex flex-wrap items-center justify-between gap-3 rounded-lg bg-muted/40 px-3 py-2 text-xs text-muted-foreground">
+        <div className="flex items-center gap-2">
+          <Settings2 className="h-3.5 w-3.5" />
+          <span>
+            {modelId
+              ? `Default: ${modelId}`
+              : "No default model configured"}
+          </span>
+        </div>
+        <div className="flex items-center gap-2">
+          {onClear && suite.defaultConfig ? (
+            <Button
+              type="button"
+              variant="ghost"
+              size="sm"
+              onClick={() => void handleClear()}
+              disabled={isClearing || isSaving}
+              className="text-destructive hover:text-destructive"
+            >
+              {isClearing ? (
+                <Loader2 className="mr-1 h-3.5 w-3.5 animate-spin" />
+              ) : (
+                <Trash2 className="mr-1 h-3.5 w-3.5" />
+              )}
+              Remove
+            </Button>
+          ) : null}
+          <Button
+            type="button"
+            variant="ghost"
+            size="sm"
+            onClick={handleReset}
+            disabled={!isDirty || isSaving || isClearing}
+          >
+            Reset
+          </Button>
+          <Button
+            type="button"
+            size="sm"
+            onClick={() => void handleSave()}
+            disabled={!isDirty || isSaving || isClearing || !modelId}
+          >
+            {isSaving ? (
+              <Loader2 className="mr-2 h-4 w-4 animate-spin" />
+            ) : null}
+            Save config
+          </Button>
+        </div>
+      </div>
+    </section>
+  );
+}

mcpjam-inspector/client/src/components/evals/suite-iterations-view.tsx

@@ -18,6 +18,7 @@ import { TestCaseDetailView } from "./test-case-detail-view";
 import { SuiteDashboard } from "./suite-dashboard";
 import { EvalExportModal } from "./eval-export-modal";
 import { SuiteEnvironmentEditor } from "./suite-environment-editor";
+import { SuiteExecutionConfigEditor } from "./suite-execution-config-editor";
 import { useSuiteData, useRunDetailData } from "./use-suite-data";
 import type {
   EvalCase,
@@ -853,6 +854,53 @@ export function SuiteIterationsView({
               />
             ) : null}
 
+            <SuiteExecutionConfigEditor
+              suite={suite}
+              availableModels={availableModels}
+              onSave={async (defaultConfig) => {
+                try {
+                  await updateSuite({
+                    suiteId: suite._id,
+                    defaultConfig,
+                  });
+                  toast.success("Suite execution config updated");
+                } catch (error) {
+                  toast.error(
+                    getBillingErrorMessage(
+                      error,
+                      "Failed to update suite execution config",
+                    ),
+                  );
+                  console.error(
+                    "Failed to update suite execution config:",
+                    error,
+                  );
+                  throw error;
+                }
+              }}
+              onClear={async () => {
+                try {
+                  await updateSuite({
+                    suiteId: suite._id,
+                    defaultConfig: null,
+                  });
+                  toast.success("Suite execution config removed");
+                } catch (error) {
+                  toast.error(
+                    getBillingErrorMessage(
+                      error,
+                      "Failed to remove suite execution config",
+                    ),
+                  );
+                  console.error(
+                    "Failed to remove suite execution config:",
+                    error,
+                  );
+                  throw error;
+                }
+              }}
+            />
+
             {/* Suite Description Section */}
             <div className="space-y-3">
               <div>

mcpjam-inspector/client/src/components/evals/types.ts

@@ -45,6 +45,12 @@ export type EvalSuite = {
   };
   _creationTime?: number; // Convex auto field
   tags?: string[];
+  defaultConfig?: {
+    modelId: string;
+    provider?: string;
+    systemPrompt: string;
+    temperature: number;
+  };
 };
 
 export type EvalCase = {