Release v0.34.0 (#294)

spboyer · Copilot · web-flow · commit 1dd4a7164638 · 2026-05-23T20:33:21.000Z
* chore: Prepare release v0.34.0

Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;

* fix: address release review feedback

Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;

---------

Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;
diff --git a/.waza.yaml b/.waza.yaml
@@ -9,7 +9,7 @@ defaults:
   model: claude-sonnet-4.6
   timeout: 300
   parallel: false
-  workers: 4
+  workers: 0
   verbose: false
   sessionLog: false
 cache:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.34.0] - 2026-05-23
+
+### Added
+
+- **BYOK provider wiring** — Added bring-your-own-key provider support for configured model providers (#240)
+- **`waza update` command** — Added an update command for upgrading local Waza installations (#288)
+- **Skill injection opt-out** — Added an option to run evals without injecting the target skill body (#285, #292)
+- **Forbidden skills grading** — `skill_invocation` graders can now assert that specific skills must not be invoked (#286, #291)
+- **Per-trial usage reporting** — Results JSON now includes per-trial usage details for deeper run analysis (#277)
+- **Agent-friendly GitHub templates** — Added issue and pull request templates tuned for agent-authored work (#293)
+
+### Fixed
+
+- **Tool approval handling** — Tool permission handling now uses the SDK approval kind (#240)
+- **Signal cancellation** — `waza run` now respects cancellation signals more reliably (#279)
+- **Sandbox prompt handling** — Empty sandbox prompts are guarded before execution (#273, #278)
+- **Custom agent example schema** — Fixed the custom-agent eval example to match the supported schema (#282)
+- **Binary release links** — Fixed binary release documentation links (#276, #284)
+- **Agent path guidance** — Corrected AGENTS root path guidance (#267, #269)
+
+### Changed
+
+- **Run concurrency** — `waza run` now reuses a shared Copilot client and auto-sizes parallel workers when `--workers` is unset (#135, #221)
+- **Documentation** — Updated integration testing, custom-agent eval, and OpenAI Evals model-graded YAML documentation (#281, #283, #14, #280)
+- **Release workflow** — GitHub Pages deployment now runs after the release workflow (#265)
+
 ## [0.33.0] - 2026-05-21
 
 Note: This release includes the changes previously prepared under 0.32.0, which was not published.
@@ -452,7 +478,8 @@ pip install waza
 - YAML escaping for regex patterns with backslashes
 - Progress bar now shows 100% on completion
 
-[Unreleased]: https://github.com/microsoft/waza/compare/v0.33.0...HEAD
+[Unreleased]: https://github.com/microsoft/waza/compare/v0.34.0...HEAD
+[0.34.0]: https://github.com/microsoft/waza/compare/v0.33.0...v0.34.0
 [0.33.0]: https://github.com/microsoft/waza/compare/v0.31.0...v0.33.0
 [0.31.0]: https://github.com/microsoft/waza/compare/v0.30.1...v0.31.0
 [0.30.1]: https://github.com/microsoft/waza/compare/v0.30.0...v0.30.1
diff --git a/Makefile b/Makefile
@@ -4,7 +4,8 @@
 # Build configuration
 BINARY_NAME=waza
 BUILD_DIR=.
-GO_FILES=$(shell find . -name '*.go' -not -path './vendor/*')
+GO_FILES=$(shell find . -name '*.go' -not -path './vendor/*' -not -path './*/node_modules/*')
+GO_PACKAGES=$(shell go list ./... | grep -v '/node_modules')
 VERSION?=0.1.0
 LDFLAGS=-ldflags "-X main.version=$(VERSION)"
 
@@ -24,14 +25,14 @@ build: build-web
 # Run tests (requires web assets for embedded FS tests)
 test: build-web
 	@echo "Running tests..."
-	@go test -v -race -coverprofile=coverage.out ./...
+	@go test -v -race -coverprofile=coverage.out $(GO_PACKAGES)
 	@go tool cover -func=coverage.out | tail -1
 
 # Run linter
 lint:
 	@echo "Running linter..."
 	@if command -v golangci-lint >/dev/null 2>&1; then \
-		golangci-lint run ./...; \
+		golangci-lint run $(GO_PACKAGES); \
 	else \
 		echo "golangci-lint not installed, skipping..."; \
 		echo "Install: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b \$$(go env GOPATH)/bin v1.64.8"; \
diff --git a/extension.yaml b/extension.yaml
@@ -4,7 +4,7 @@ namespace: waza
 displayName: Waza - AI Agent Skill Evaluator
 description: A CLI for evaluating AI agent skills — scaffold eval suites, run benchmarks, and compare results across models.
 usage: azd waza <command> [options]
-version: 0.33.0
+version: 0.34.0
 language: go
 capabilities:
   - custom-commands
diff --git a/internal/validation/doc_examples_test.go b/internal/validation/doc_examples_test.go
@@ -49,7 +49,7 @@ config:
   trials_per_task: 1
   timeout_seconds: 300
   parallel: false
-  workers: 4
+  workers: 0
   model: claude-sonnet-4.6
   judge_model: gpt-4o
   executor: mock
diff --git a/schemas/eval.schema.json b/schemas/eval.schema.json
@@ -118,8 +118,8 @@
         },
         "workers": {
           "type": "integer",
-          "minimum": 1,
-          "description": "Maximum number of concurrent workers when parallel is true."
+          "minimum": 0,
+          "description": "Maximum number of concurrent workers when parallel is true. Use 0 or omit to auto-size workers."
         },
         "fail_fast": {
           "type": "boolean",
diff --git a/site/src/content/docs/getting-started.mdx b/site/src/content/docs/getting-started.mdx
@@ -284,7 +284,7 @@ Opens http://localhost:3000. View runs, detailed results, comparisons, and trend
 If the install script exits with a download error or 404, check the [Releases page](https://github.com/microsoft/waza/releases/latest) to confirm a binary for your platform is available. You can also install manually:
 
 ```bash
-# Example: macOS ARM64 (Apple Silicon) — replace <version> with the latest tag (e.g. v0.33.0)
+# Example: macOS ARM64 (Apple Silicon) — replace <version> with the latest tag (e.g. v0.34.0)
 curl -fSL -o waza https://github.com/microsoft/waza/releases/download/<version>/waza-darwin-arm64
 chmod +x waza
 sudo mv waza /usr/local/bin/waza
diff --git a/site/src/content/docs/guides/eval-yaml.mdx b/site/src/content/docs/guides/eval-yaml.mdx
@@ -94,7 +94,7 @@ config:
   trials_per_task: 1 # Run each task this many times
   timeout_seconds: 300 # Task timeout in seconds
   parallel: false # Run tasks sequentially (true = concurrent)
-  workers: 4 # Parallel workers if parallel: true
+  workers: 0 # Auto-size parallel workers if parallel: true
   model: claude-sonnet-4.6 # Default model (override with --model)
   judge_model: gpt-4o # Model for LLM-as-judge graders (optional)
   executor: mock # mock (local) or copilot-sdk (real API)
@@ -108,7 +108,7 @@ config:
 | `trials_per_task`   | int       | 1                 | Number of times each task runs (for statistical analysis)   |
 | `timeout_seconds`   | int       | 300               | Task timeout in seconds                                     |
 | `parallel`          | bool      | false             | Run tasks concurrently                                      |
-| `workers`           | int       | 4                 | Number of parallel workers                                  |
+| `workers`           | int       | 0                 | Number of parallel workers; 0 auto-sizes                    |
 | `model`             | string    | _required_        | Default model for tasks (override with `--model` flag)      |
 | `judge_model`       | string    | (same as `model`) | Model for `prompt`-type graders (LLM-as-judge)              |
 | `executor`          | string    | `copilot-sdk`     | Executor: `mock` (local, echoes task metadata and file content) or `copilot-sdk` (real API)  |
@@ -555,8 +555,8 @@ waza run eval.yaml --tags "happy-path"
 ### Parallel Execution
 
 ```bash
-# Run tasks concurrently with 4 workers
-waza run eval.yaml --parallel --workers 4
+# Run tasks concurrently with auto-sized workers
+waza run eval.yaml --parallel
 ```
 
 ## Saving Results
diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx
@@ -64,7 +64,7 @@ waza run [eval.yaml | skill-name | agent-name]
 | `--output-dir` | `-d` | string | | Directory for structured output; each run creates a UTC-timestamped subdirectory of the specified directory. Mutually exclusive with `--output`. |
 | `--verbose` | `-v` | bool | false | Detailed progress output |
 | `--parallel` | | bool | false | Run tasks concurrently |
-| `--workers` | `-w` | int | 4 | Number of concurrent workers |
+| `--workers` | `-w` | int | auto | Number of concurrent workers |
 | `--trials` | | int | `config.trials_per_task` | Run each task N times for flakiness detection (omit to use `config.trials_per_task`; when provided, value must be >= 1) |
 | `--task` | `-t` | string | | Filter tasks by name (repeatable) |
 | `--tags` | | string | | Filter tasks by tags (repeatable) |
diff --git a/site/src/content/docs/reference/releases.mdx b/site/src/content/docs/reference/releases.mdx
@@ -5,28 +5,31 @@ description: Download waza binaries and the azd extension. Changelog highlights
 
 import { Tabs, TabItem } from '@astrojs/starlight/components';
 
-## Current Release: v0.33.0
+## Current Release: v0.34.0
 
-**Released:** 2026-05-21
+**Released:** 2026-05-23
 
 ### What's New
 
 #### Added
 
-- **Configurable eval file naming** — Projects can configure generated and discovered eval filenames through `.waza.yaml` `files.evalFile`, `files.taskGlob`, and `files.taskFileSuffix`.
-- **Instruction files in eval runs** — Eval-level and task-level instruction files can be copied into isolated workspaces and injected into the Copilot system message.
+- **BYOK provider wiring** — Bring-your-own-key provider support is wired for configured model providers.
+- **`waza update` command** — Local Waza installations can be upgraded from the CLI.
+- **Skill injection opt-out** — Evals can run without injecting the target skill body.
+- **Forbidden skills grading** — `skill_invocation` graders can assert that specific skills must not be invoked.
+- **Per-trial usage reporting** — Results JSON includes per-trial usage details for deeper run analysis.
 
 #### Fixed
 
-- **Prompt graders use the execution engine** — Judge turns now route through `CopilotEngine`, keeping grader execution aligned with engine configuration.
-- **Prompt grader follow-up recovery** — Prompt grading preserves collected grades when a follow-up turn fails after successful grader collection.
-- **Bundled Copilot CLI updated** — Embedded `copilot-cli` bundles are updated to 1.0.49.
-- **Spec-aligned skill scaffolding** — `waza new skill` no longer asks for a nonstandard skill type or emits `type:` frontmatter.
-- **Improved discovery and routing checks** — `waza check` discovers nested/separated evals consistently and compliance scoring detects routing markers in `SKILL.md` body sections.
+- **Tool approval handling** — Tool permissions use the SDK approval kind.
+- **Signal cancellation** — `waza run` respects cancellation signals more reliably.
+- **Sandbox prompt handling** — Empty sandbox prompts are guarded before execution.
+- **Custom agent example schema** — The custom-agent eval example now matches the supported schema.
 
 #### Changed
 
-- **Copilot SDK v0.3.0 migration** — Session events, transcript handling, logging, web API usage collection, suggestion traces, and tests now use the updated SDK API.
+- **Run concurrency** — `waza run` reuses a shared Copilot client and auto-sizes parallel workers when `--workers` is unset.
+- **Documentation** — Integration testing, custom-agent eval, and OpenAI Evals model-graded YAML docs were refreshed.
 
 ---
 
@@ -36,12 +39,12 @@ Pre-built binaries for all major platforms:
 
 | Platform | Architecture | Download |
 |----------|-------------|----------|
-| macOS | Apple Silicon (ARM64) | [waza-darwin-arm64](https://github.com/microsoft/waza/releases/download/v0.33.0/waza-darwin-arm64) |
-| macOS | Intel (AMD64) | [waza-darwin-amd64](https://github.com/microsoft/waza/releases/download/v0.33.0/waza-darwin-amd64) |
-| Linux | x64 (AMD64) | [waza-linux-amd64](https://github.com/microsoft/waza/releases/download/v0.33.0/waza-linux-amd64) |
-| Linux | ARM64 | [waza-linux-arm64](https://github.com/microsoft/waza/releases/download/v0.33.0/waza-linux-arm64) |
-| Windows | x64 (AMD64) | [waza-windows-amd64.exe](https://github.com/microsoft/waza/releases/download/v0.33.0/waza-windows-amd64.exe) |
-| Windows | ARM64 | [waza-windows-arm64.exe](https://github.com/microsoft/waza/releases/download/v0.33.0/waza-windows-arm64.exe) |
+| macOS | Apple Silicon (ARM64) | [waza-darwin-arm64](https://github.com/microsoft/waza/releases/download/v0.34.0/waza-darwin-arm64) |
+| macOS | Intel (AMD64) | [waza-darwin-amd64](https://github.com/microsoft/waza/releases/download/v0.34.0/waza-darwin-amd64) |
+| Linux | x64 (AMD64) | [waza-linux-amd64](https://github.com/microsoft/waza/releases/download/v0.34.0/waza-linux-amd64) |
+| Linux | ARM64 | [waza-linux-arm64](https://github.com/microsoft/waza/releases/download/v0.34.0/waza-linux-arm64) |
+| Windows | x64 (AMD64) | [waza-windows-amd64.exe](https://github.com/microsoft/waza/releases/download/v0.34.0/waza-windows-amd64.exe) |
+| Windows | ARM64 | [waza-windows-arm64.exe](https://github.com/microsoft/waza/releases/download/v0.34.0/waza-windows-arm64.exe) |
 
 ---
 
@@ -63,7 +66,7 @@ On Windows, use this command only from Git Bash, MSYS2, or Cygwin. If PowerShell
 irm https://raw.githubusercontent.com/microsoft/waza/main/install.ps1 | iex
 ```
 
-The PowerShell installer downloads `waza-windows-amd64.exe` or `waza-windows-arm64.exe`, verifies the checksum, and installs `waza.exe`. You can also download the Windows binary from the [standalone waza release](https://github.com/microsoft/waza/releases/tag/v0.33.0), rename it to `waza.exe`, and place it in a directory on your `PATH`.
+The PowerShell installer downloads `waza-windows-amd64.exe` or `waza-windows-arm64.exe`, verifies the checksum, and installs `waza.exe`. You can also download the Windows binary from the [standalone waza release](https://github.com/microsoft/waza/releases/tag/v0.34.0), rename it to `waza.exe`, and place it in a directory on your `PATH`.
 
 </TabItem>
 </Tabs>
@@ -80,11 +83,11 @@ waza --version
 
 Install waza as an [Azure Developer CLI](https://learn.microsoft.com/azure/developer/azure-developer-cli/) extension.
 
-**Current version:** v0.33.0
+**Current version:** v0.34.0
 
 ```bash
 azd extension install azd-waza \
-  --source https://github.com/microsoft/waza/releases/download/azd-ext-microsoft-azd-waza_0.33.0/azd-ext-microsoft-azd-waza_0.33.0.tar.gz
+  --source https://github.com/microsoft/waza/releases/download/azd-ext-microsoft-azd-waza_0.34.0/azd-ext-microsoft-azd-waza_0.34.0.tar.gz
 ```
 
 Once installed, run evaluations through `azd`:
diff --git a/site/src/content/docs/reference/schema.mdx b/site/src/content/docs/reference/schema.mdx
@@ -21,7 +21,7 @@ config:
   trials_per_task: 1 # Runs per task
   timeout_seconds: 300 # Task timeout
   parallel: false # Concurrent execution
-  workers: 4 # Parallel workers
+  workers: 0 # Auto-size parallel workers
   model: claude-sonnet-4.6 # Default model
   executor: mock # mock or copilot-sdk
 
@@ -130,9 +130,9 @@ config:
 ### workers
 
 **Type:** integer  
-**Default:** 4
+**Default:** 0
 
-Number of concurrent workers when `parallel: true`.
+Number of concurrent workers when `parallel: true`. Use `0` or omit the field to auto-size workers.
 
 ```yaml
 config:
@@ -653,7 +653,7 @@ files:
 defaults:
   model: claude-sonnet-4.6
   timeout: 300
-  workers: 4
+  workers: 0
 
 # Cache settings
 cache:
@@ -779,7 +779,7 @@ config:
   trials_per_task: 1
   timeout_seconds: 300
   parallel: false
-  workers: 4
+  workers: 0
   model: claude-sonnet-4.6
   executor: copilot-sdk
 
diff --git a/site/src/content/docs/reference/waza-yaml.mdx b/site/src/content/docs/reference/waza-yaml.mdx
@@ -22,7 +22,7 @@ files:
 defaults:
   model: claude-sonnet-4.6
   timeout: 300
-  workers: 4
+  workers: 0
 ```
 
 ## Configuration Sections
@@ -65,7 +65,7 @@ Default execution parameters applied to all commands unless overridden by CLI fl
 | `judgeModel` | string | | Model for LLM-as-judge graders |
 | `timeout` | int | `300` | Task timeout in seconds |
 | `parallel` | bool | `false` | Enable parallel execution |
-| `workers` | int | `4` | Number of parallel workers |
+| `workers` | int | `0` | Number of parallel workers; 0 auto-sizes |
 | `verbose` | bool | `false` | Enable verbose output |
 | `sessionLog` | bool | `false` | Enable session event logging |
 
@@ -175,7 +175,7 @@ defaults:
   model: claude-sonnet-4.6
   timeout: 300
   parallel: false
-  workers: 4
+  workers: 0
 
 cache:
   enabled: false
diff --git a/version.txt b/version.txt
@@ -1 +1 @@
-0.33.0
+0.34.0
diff --git a/waza-runner/references/EVAL-SPEC.md b/waza-runner/references/EVAL-SPEC.md
@@ -22,7 +22,7 @@ config:
   trials_per_task: 3      # Number of times to run each task
   timeout_seconds: 300    # Max time per trial
   parallel: false         # Run tasks concurrently
-  workers: 4          # Max parallel workers
+  workers: 0              # Auto-size parallel workers
   fail_fast: false        # Stop on first failure
   verbose: false          # Verbose output
 ```
@@ -90,7 +90,7 @@ config:
   trials_per_task: 3
   timeout_seconds: 300
   parallel: true
-  workers: 4
+  workers: 0
 
 metrics:
   - name: task_completion
