feat: add openai-compatible executor

README.md

@@ -893,8 +893,10 @@ config:
   max_attempts: 3          # Retry failed graders up to 3 times (default: 1, no retries)
   timeout_seconds: 300
   parallel: false
-  executor: mock          # or copilot-sdk
+  executor: mock          # or copilot-sdk, openai-compatible
   model: claude-sonnet-4-20250514
+  endpoint: http://127.0.0.1:1234 # Required for openai-compatible unless set in .waza.yaml
+  api_key: ""             # Optional per-eval override; prefer OPENAI_API_KEY when auth is required
   group_by: model          # Group results by model (or other dimension)
   instruction_files:
     - .github/instructions/project.instructions.md
@@ -1250,6 +1252,7 @@ jobs:
 | **Go Version** | 1.26 or higher |
 | **Executor** | Use `mock` executor for CI (no API keys needed) |
 | **Copilot Auth** | Required for the default `copilot-sdk` route; set `GITHUB_TOKEN` in CI. Custom providers can be configured with `COPILOT_BASE_URL` or `COPILOT_PROVIDER_BASE_URL` instead. |
+| **OpenAI-compatible endpoint** | Required for `openai-compatible` unless set as `.waza.yaml` `defaults.endpoint`; for local LM Studio use `http://127.0.0.1:1234`. |
 | **Exit Codes** | 0=success, 1=test failure, 2=config error |
 
 #### Expected Skill Structure
@@ -1288,9 +1291,34 @@ Supported environment variables:
 | `COPILOT_WIRE_API` or `COPILOT_PROVIDER_WIRE_API` | Wire format passed through to the SDK, for example `responses` or `completions`, depending on provider. |
 | `COPILOT_API_KEY` or `COPILOT_PROVIDER_API_KEY` | API key for the custom provider, if required. |
 | `COPILOT_BEARER_TOKEN` or `COPILOT_PROVIDER_BEARER_TOKEN` | Bearer token for the custom provider, if required. |
+| `OPENAI_API_KEY` | Bearer token sent by the `openai-compatible` executor when the endpoint requires authentication. |
 
 When a custom provider is active, the CLI usage summary labels the SDK request counter as `Provider Requests` instead of `Premium Requests`. Result JSON records `usage.provider: "custom"` and a sanitized `usage.provider_host`; it does not store the full provider URL.
 
+#### OpenAI-Compatible Executor
+
+Use `executor: openai-compatible` to run directly against a local or hosted OpenAI-compatible Chat Completions API without the Copilot SDK. The `endpoint` may be a base URL, `/v1` URL, or full `/v1/chat/completions` URL. `model` is optional and defaults to `local-model`.
+
+```yaml
+config:
+  trials_per_task: 1
+  timeout_seconds: 300
+  executor: openai-compatible
+  endpoint: http://127.0.0.1:1234
+  model: local-model
+  api_key: "" # Optional per-eval override; prefer OPENAI_API_KEY when auth is required
+```
+
+You can also put the executor and endpoint in `.waza.yaml`:
+
+```yaml
+defaults:
+  engine: openai-compatible
+  endpoint: http://127.0.0.1:1234
+```
+
+When the endpoint requires bearer-token authentication, set `OPENAI_API_KEY`. `config.api_key` remains available as a per-eval override, but project-level config does not store API keys.
+
 ### For Waza Repository
 
 This repository includes reusable workflows:

cmd/waza/cmd_run.go

@@ -473,6 +473,9 @@ func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []str
 	if err != nil {
 		return nil, fmt.Errorf("failed to load spec: %w", err)
 	}
+	if cfg, err := projectconfig.Load(filepath.Dir(specPath)); err == nil && cfg != nil {
+		applyProjectDefaultsToSpec(spec, cfg)
+	}
 
 	// CLI flags override spec config
 	if parallel {
@@ -594,6 +597,21 @@ func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []str
 	return allResults, nil
 }
 
+func applyProjectDefaultsToSpec(spec *models.EvalSpec, cfg *projectconfig.ProjectConfig) {
+	if spec == nil || cfg == nil {
+		return
+	}
+	if spec.Config.EngineType == "" {
+		spec.Config.EngineType = cfg.Defaults.Engine
+	}
+	if spec.Config.ModelID == "" && (spec.Config.EngineType != "openai-compatible" || cfg.Defaults.Model != projectconfig.DefaultModel) {
+		spec.Config.ModelID = cfg.Defaults.Model
+	}
+	if spec.Config.Endpoint == "" {
+		spec.Config.Endpoint = cfg.Defaults.Endpoint
+	}
+}
+
 // runSingleModel executes a benchmark for one model and returns the outcome.
 // It prints the per-model summary and saves output for single-model runs.
 func runSingleModel(cmd *cobra.Command, spec *models.EvalSpec, specPath string, defaultSkills []string) (*models.EvaluationOutcome, error) {
@@ -655,6 +673,7 @@ func runSingleModel(cmd *cobra.Command, spec *models.EvalSpec, specPath string,
 
 	// Create engine based on spec
 	var engine execution.AgentEngine
+	var err error
 
 	switch spec.Config.EngineType {
 	case "mock":
@@ -663,6 +682,11 @@ func runSingleModel(cmd *cobra.Command, spec *models.EvalSpec, specPath string,
 		engine = execution.NewCopilotEngineBuilder(spec.Config.ModelID, &execution.CopilotEngineBuilderOptions{
 			NewCopilotClient: newCopilotClientFn, // if nil, uses the real function, otherwise overridable for tests.
 		}).Build()
+	case "openai-compatible":
+		engine, err = execution.NewOpenAICompatibleEngine(spec.Config.Endpoint, spec.Config.ModelID, spec.Config.APIKey)
+		if err != nil {
+			return nil, err
+		}
 	default:
 		return nil, fmt.Errorf("unknown engine type: %s", spec.Config.EngineType)
 	}

cmd/waza/cmd_run_test.go

@@ -9,6 +9,8 @@ import (
 	"io"
 	"io/fs"
 	"maps"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"slices"
@@ -716,6 +718,113 @@ tasks:
 	assert.Contains(t, err.Error(), "unknown engine type")
 }
 
+func TestRunCommand_OpenAICompatibleEngine(t *testing.T) {
+	resetRunGlobals()
+
+	var hitPath string
+	var gotAuth string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hitPath = r.URL.Path
+		gotAuth = r.Header.Get("Authorization")
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"openai compatible response"}}]}`))
+	}))
+	defer server.Close()
+
+	dir := t.TempDir()
+	taskDir := filepath.Join(dir, "tasks")
+	require.NoError(t, os.MkdirAll(taskDir, 0o755))
+	task := `id: t1
+name: OpenAI Compatible Task
+inputs:
+  prompt: "Say hello"
+expected:
+  output_contains:
+    - "openai compatible response"
+`
+	require.NoError(t, os.WriteFile(filepath.Join(taskDir, "task.yaml"), []byte(task), 0o644))
+	spec := `name: openai-compatible-test
+skill: test-skill
+config:
+  trials_per_task: 1
+  timeout_seconds: 30
+  executor: openai-compatible
+  endpoint: ` + server.URL + `
+  api_key: lm-studio
+tasks:
+  - "tasks/*.yaml"
+`
+	specPath := filepath.Join(dir, "eval.yaml")
+	require.NoError(t, os.WriteFile(specPath, []byte(spec), 0o644))
+
+	cmd := newRunCommand()
+	cmd.SetArgs([]string{specPath})
+	cmd.SetOut(io.Discard)
+	cmd.SetErr(io.Discard)
+
+	err := cmd.Execute()
+	require.NoError(t, err)
+	require.Equal(t, "/v1/chat/completions", hitPath)
+	require.Equal(t, "Bearer lm-studio", gotAuth)
+}
+
+func TestRunCommand_OpenAICompatibleDefaultsFromWazaYaml(t *testing.T) {
+	resetRunGlobals()
+
+	var hitPath string
+	var gotBody struct {
+		Model string `json:"model"`
+	}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hitPath = r.URL.Path
+		require.Empty(t, r.Header.Get("Authorization"))
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&gotBody))
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"defaulted response"}}]}`))
+	}))
+	defer server.Close()
+
+	dir := t.TempDir()
+	wazaYAML := `defaults:
+  engine: openai-compatible
+  endpoint: ` + server.URL + `
+`
+	require.NoError(t, os.WriteFile(filepath.Join(dir, ".waza.yaml"), []byte(wazaYAML), 0o644))
+
+	taskDir := filepath.Join(dir, "tasks")
+	require.NoError(t, os.MkdirAll(taskDir, 0o755))
+	task := `id: t1
+name: OpenAI Compatible Defaults Task
+inputs:
+  prompt: "Say hello"
+expected:
+  output_contains:
+    - "defaulted response"
+`
+	require.NoError(t, os.WriteFile(filepath.Join(taskDir, "task.yaml"), []byte(task), 0o644))
+	spec := `name: openai-compatible-defaults-test
+skill: test-skill
+config:
+  trials_per_task: 1
+  timeout_seconds: 30
+tasks:
+  - "tasks/*.yaml"
+`
+	specPath := filepath.Join(dir, "eval.yaml")
+	require.NoError(t, os.WriteFile(specPath, []byte(spec), 0o644))
+	t.Setenv("OPENAI_API_KEY", "")
+
+	cmd := newRunCommand()
+	cmd.SetArgs([]string{specPath})
+	cmd.SetOut(io.Discard)
+	cmd.SetErr(io.Discard)
+
+	err := cmd.Execute()
+	require.NoError(t, err)
+	require.Equal(t, "/v1/chat/completions", hitPath)
+	require.Equal(t, "local-model", gotBody.Model)
+}
+
 // ---------------------------------------------------------------------------
 // --format flag
 // ---------------------------------------------------------------------------

internal/execution/copilot.go

@@ -628,6 +628,10 @@ func (e *CopilotEngine) extractReqParams(req *ExecutionRequest) (modelID string,
 }
 
 func (*CopilotEngine) getSkillDirs(cwd string, req *ExecutionRequest) []string {
+	return skillDirsForRequest(cwd, req)
+}
+
+func skillDirsForRequest(cwd string, req *ExecutionRequest) []string {
 	skillDirs := []string{cwd}
 
 	seen := map[string]bool{