support more models for llm-as-judge functionality, expose what models/providers are available based on API keys in ENV vars

Author: krisztianfekete

src/agentevals/api/routes.py

@@ -28,6 +28,18 @@ async def health_check():
     return {"status": "ok", "version": __version__}
 
 
+@router.get("/config")
+async def get_config():
+    """Return environment configuration, including which API keys are available."""
+    return {
+        "apiKeys": {
+            "google": bool(os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY")),
+            "anthropic": bool(os.environ.get("ANTHROPIC_API_KEY")),
+            "openai": bool(os.environ.get("OPENAI_API_KEY")),
+        }
+    }
+
+
 @router.get("/metrics")
 async def list_metrics():
     """List available metrics with metadata.

ui/src/api/client.ts

@@ -215,6 +215,14 @@ export async function validateEvalSet(evalSetFile: File) {
   }
 }
 
+export async function getConfig(): Promise<{ apiKeys: { google: boolean; anthropic: boolean; openai: boolean } }> {
+  const response = await fetch(`${API_BASE_URL}/config`);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch config: ${response.statusText}`);
+  }
+  return response.json();
+}
+
 /**
  * Health check
  */

ui/src/components/upload/UploadView.tsx

@@ -173,11 +173,13 @@ const uploadViewStyle = css`
 `;
 
 const JUDGE_MODELS = [
-  'gemini-2.5-flash',
-  'gemini-2.0-flash',
-  'claude-3.5-sonnet',
-  'gpt-4o',
-];
+  { value: 'gemini-2.5-flash', provider: 'google' },
+  { value: 'gemini-2.0-flash', provider: 'google' },
+  { value: 'anthropic/claude-3.5-sonnet', provider: 'anthropic' },
+  { value: 'openai/gpt-4o', provider: 'openai' },
+] as const;
+
+type JudgeModelProvider = typeof JUDGE_MODELS[number]['provider'];
 
 export const UploadView: React.FC = () => {
   const { state, actions } = useTraceContext();
@@ -430,13 +432,40 @@ export const UploadView: React.FC = () => {
             <Select
               value={state.judgeModel}
               onChange={actions.setJudgeModel}
-              options={JUDGE_MODELS.map((model) => ({ label: model, value: model }))}
+              options={JUDGE_MODELS.map(({ value }) => ({ label: value, value }))}
               style={{ width: '100%' }}
               size="small"
             />
-            <span className="setting-hint">
-              LLM for judge-based metrics
-            </span>
+            {state.apiKeyStatus && (
+              <div style={{ display: 'flex', gap: 10, marginTop: 4, flexWrap: 'wrap' }}>
+                {([
+                  { label: 'GOOGLE_API_KEY', provider: 'google' as const },
+                  { label: 'ANTHROPIC_API_KEY', provider: 'anthropic' as const },
+                  { label: 'OPENAI_API_KEY', provider: 'openai' as const },
+                ] as const).map(({ label, provider }) => {
+                  const ok = state.apiKeyStatus![provider];
+                  return (
+                    <span key={provider} style={{ display: 'flex', alignItems: 'center', gap: 4, fontSize: 10, color: ok ? 'var(--status-success)' : 'var(--status-failure)' }}>
+                      <span style={{ width: 6, height: 6, borderRadius: '50%', backgroundColor: ok ? 'var(--status-success)' : 'var(--status-failure)', flexShrink: 0 }} />
+                      {label}
+                    </span>
+                  );
+                })}
+              </div>
+            )}
+            {(() => {
+              const selected = JUDGE_MODELS.find((m) => m.value === state.judgeModel);
+              const provider = selected?.provider as JudgeModelProvider | undefined;
+              if (!provider || !state.apiKeyStatus || state.apiKeyStatus[provider]) return null;
+              const keyName = provider === 'google' ? 'GOOGLE_API_KEY / GEMINI_API_KEY'
+                : provider === 'anthropic' ? 'ANTHROPIC_API_KEY'
+                : 'OPENAI_API_KEY';
+              return (
+                <span style={{ fontSize: '11px', color: 'var(--status-failure)', marginTop: 2 }}>
+                  {keyName} is not set — this model will fail
+                </span>
+              );
+            })()}
           </div>
 
           <div className="setting-item" style={{ marginTop: 10 }}>

ui/src/context/TraceContext.tsx

@@ -2,6 +2,12 @@ import { createContext, useContext } from 'react';
 import type { TraceResult, ViewType, EvalSet, EvalSetMetadata, EvalCase, TraceTableRow, LiveSession, AnnotationQueue, Annotation } from '../lib/types';
 import type { TraceMetadata } from '../lib/trace-metadata';
 
+export interface ApiKeyStatus {
+  google: boolean;
+  anthropic: boolean;
+  openai: boolean;
+}
+
 export interface TraceState {
   // Upload state
   traceFiles: File[];
@@ -11,6 +17,7 @@ export interface TraceState {
   threshold: number;
   traceMetadata: Map<string, TraceMetadata>;
   isLoadingMetadata: boolean;
+  apiKeyStatus: ApiKeyStatus | null;
 
   // Evaluation state
   isEvaluating: boolean;

ui/src/context/TraceProvider.tsx

@@ -1,9 +1,9 @@
-import React, { useState, useMemo } from 'react';
+import React, { useState, useMemo, useEffect } from 'react';
 import type { ReactNode } from 'react';
 import { TraceContext } from './TraceContext';
 import type { TraceState } from './TraceContext';
 import type { ViewType, EvalSet, EvalSetMetadata, EvalCase, LiveSession, AnnotationQueue, Annotation } from '../lib/types';
-import { evaluateTracesStreaming } from '../api/client';
+import { evaluateTracesStreaming, getConfig } from '../api/client';
 import { extractMetadataFromTraceFile } from '../lib/trace-metadata';
 
 interface TraceProviderProps {
@@ -19,6 +19,7 @@ export const TraceProvider: React.FC<TraceProviderProps> = ({ children }) => {
     threshold: 0.8,
     traceMetadata: new Map(),
     isLoadingMetadata: false,
+    apiKeyStatus: null,
     isEvaluating: false,
     progressMessage: '',
     results: [],
@@ -37,6 +38,12 @@ export const TraceProvider: React.FC<TraceProviderProps> = ({ children }) => {
     builderSelectedTraceIds: [],
   });
 
+  useEffect(() => {
+    getConfig()
+      .then((cfg) => setState((prev) => ({ ...prev, apiKeyStatus: cfg.apiKeys })))
+      .catch(() => {}); // silently ignore if backend is unreachable
+  }, []);
+
   const actions = useMemo(
     () => ({
       setTraceFiles: async (files: File[]) => {