Merge branch 'main' into wr/activateVersion

Author: WillieRuemmele

CHANGELOG.md

@@ -1,3 +1,33 @@
+## [1.31.3](https://github.com/salesforcecli/plugin-agent/compare/1.31.2...1.31.3) (2026-03-08)
+
+### Bug Fixes
+
+- **deps:** bump fast-xml-parser from 5.4.1 to 5.4.2 ([91a1ff8](https://github.com/salesforcecli/plugin-agent/commit/91a1ff8124ed56b86c421077067d0e5f4a6ccee1))
+
+## [1.31.2](https://github.com/salesforcecli/plugin-agent/compare/1.31.1...1.31.2) (2026-03-08)
+
+### Bug Fixes
+
+- **deps:** bump @salesforce/core from 8.26.2 to 8.26.3 ([033b12c](https://github.com/salesforcecli/plugin-agent/commit/033b12cc8fab8a94091f50b524cc02d70a165861))
+
+## [1.31.1](https://github.com/salesforcecli/plugin-agent/compare/1.31.0...1.31.1) (2026-03-08)
+
+### Bug Fixes
+
+- **deps:** bump @salesforce/agents from 0.23.4 to 0.24.1 ([229e94c](https://github.com/salesforcecli/plugin-agent/commit/229e94cb85adb6bd910f4175a3c19a09c5610ce2))
+
+# [1.31.0](https://github.com/salesforcecli/plugin-agent/compare/1.30.11...1.31.0) (2026-03-06)
+
+### Bug Fixes
+
+- initial updates, use Org/Connection, make beta/hidden ([2a0f283](https://github.com/salesforcecli/plugin-agent/commit/2a0f28396cd31ae9c8069a303501d77d5eac6bac))
+
+### Features
+
+- add `sf agent test run-eval` for Einstein Eval Labs API testing ([1699d51](https://github.com/salesforcecli/plugin-agent/commit/1699d5188a3e4b165c564680e65e5dc33b20bb6e))
+- add YAML test spec support for `sf agent test run-eval` ([e942e00](https://github.com/salesforcecli/plugin-agent/commit/e942e006e7892d34ae93e6f84a6c6673b8be0c9b))
+- normalize MCP shorthand format in evalNormalizer ([01bc87b](https://github.com/salesforcecli/plugin-agent/commit/01bc87be643b4901ba65d79b205a1977f750c81a))
+
 ## [1.30.11](https://github.com/salesforcecli/plugin-agent/compare/1.30.10...1.30.11) (2026-03-01)
 
 ### Bug Fixes

README.md

@@ -116,7 +116,7 @@ EXAMPLES
     $ sf agent activate --api-name Resort_Manager --target-org my-org
 ```
 
-_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/activate.ts)_
+_See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/activate.ts)_
 
 ## `sf agent create`
 
@@ -183,7 +183,7 @@ EXAMPLES
     $ sf agent create --name "Resort Manager" --spec specs/resortManagerAgent.yaml --preview
 ```
 
-_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/create.ts)_
+_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/create.ts)_
 
 ## `sf agent deactivate`
 
@@ -223,7 +223,7 @@ EXAMPLES
     $ sf agent deactivate --api-name Resort_Manager --target-org my-org
 ```
 
-_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/deactivate.ts)_
+_See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/deactivate.ts)_
 
 ## `sf agent generate agent-spec`
 
@@ -330,7 +330,7 @@ EXAMPLES
     $ sf agent generate agent-spec --tone formal --agent-user resortmanager@myorg.com
 ```
 
-_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/generate/agent-spec.ts)_
+_See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/generate/agent-spec.ts)_
 
 ## `sf agent generate authoring-bundle`
 
@@ -407,7 +407,7 @@ EXAMPLES
       other-package-dir/main/default --target-org my-dev-org
 ```
 
-_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/generate/authoring-bundle.ts)_
+_See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/generate/authoring-bundle.ts)_
 
 ## `sf agent generate template`
 
@@ -455,7 +455,7 @@ EXAMPLES
       force-app/main/default/bots/My_Awesome_Agent/My_Awesome_Agent.bot-meta.xml --agent-version 1
 ```
 
-_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/generate/template.ts)_
+_See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/generate/template.ts)_
 
 ## `sf agent generate test-spec`
 
@@ -516,7 +516,7 @@ EXAMPLES
       force-app//main/default/aiEvaluationDefinitions/Resort_Manager_Tests.aiEvaluationDefinition-meta.xml
 ```
 
-_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/generate/test-spec.ts)_
+_See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/generate/test-spec.ts)_
 
 ## `sf agent preview`
 
@@ -589,7 +589,7 @@ EXAMPLES
     $ sf agent preview --use-live-actions --apex-debug --output-dir transcripts/my-preview
 ```
 
-_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview.ts)_
+_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview.ts)_
 
 ## `sf agent preview end`
 
@@ -644,7 +644,7 @@ EXAMPLES
     $ sf agent preview end --authoring-bundle My_Local_Agent
 ```
 
-_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview/end.ts)_
+_See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview/end.ts)_
 
 ## `sf agent preview send`
 
@@ -702,7 +702,7 @@ EXAMPLES
     $ sf agent preview send --utterance "what can you help me with?" --authoring-bundle My_Local_Agent
 ```
 
-_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview/send.ts)_
+_See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview/send.ts)_
 
 ## `sf agent preview sessions`
 
@@ -735,7 +735,7 @@ EXAMPLES
     $ sf agent preview sessions
 ```
 
-_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview/sessions.ts)_
+_See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview/sessions.ts)_
 
 ## `sf agent preview start`
 
@@ -792,7 +792,7 @@ EXAMPLES
     $ sf agent preview start --api-name My_Published_Agent
 ```
 
-_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/preview/start.ts)_
+_See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview/start.ts)_
 
 ## `sf agent publish authoring-bundle`
 
@@ -841,7 +841,7 @@ EXAMPLES
     $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --target-org my-dev-org
 ```
 
-_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/publish/authoring-bundle.ts)_
+_See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/publish/authoring-bundle.ts)_
 
 ## `sf agent test create`
 
@@ -896,7 +896,7 @@ EXAMPLES
     $ sf agent test create --spec specs/Resort_Manager-testSpec.yaml --api-name Resort_Manager_Test --preview
 ```
 
-_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/create.ts)_
+_See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/create.ts)_
 
 ## `sf agent test list`
 
@@ -931,7 +931,7 @@ EXAMPLES
     $ sf agent test list --target-org my-org
 ```
 
-_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/list.ts)_
+_See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/list.ts)_
 
 ## `sf agent test results`
 
@@ -997,7 +997,7 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/results.ts)_
+_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/results.ts)_
 
 ## `sf agent test resume`
 
@@ -1070,7 +1070,7 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/resume.ts)_
+_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/resume.ts)_
 
 ## `sf agent test run`
 
@@ -1144,7 +1144,7 @@ FLAG DESCRIPTIONS
     expression when using custom evaluations.
 ```
 
-_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/test/run.ts)_
+_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/run.ts)_
 
 ## `sf agent validate authoring-bundle`
 
@@ -1191,6 +1191,6 @@ EXAMPLES
     $ sf agent validate authoring-bundle --api-name MyAuthoringBundle --target-org my-dev-org
 ```
 
-_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.30.11/src/commands/agent/validate/authoring-bundle.ts)_
+_See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/validate/authoring-bundle.ts)_
 
 <!-- commandsstop -->

command-snapshot.json

@@ -213,6 +213,25 @@
     ],
     "plugin": "@salesforce/plugin-agent"
   },
+  {
+    "alias": [],
+    "command": "agent:test:run-eval",
+    "flagAliases": [],
+    "flagChars": ["n", "o", "s", "w"],
+    "flags": [
+      "api-name",
+      "api-version",
+      "batch-size",
+      "flags-dir",
+      "json",
+      "no-normalize",
+      "result-format",
+      "spec",
+      "target-org",
+      "wait"
+    ],
+    "plugin": "@salesforce/plugin-agent"
+  },
   {
     "alias": [],
     "command": "agent:validate:authoring-bundle",

messages/agent.test.run-eval.md

@@ -0,0 +1,91 @@
+# summary
+
+Run evaluation tests against an Agentforce agent.
+
+# description
+
+Execute rich evaluation tests against an Agentforce agent using the Einstein Evaluation API. Supports both YAML test specs (same format as `sf agent generate test-spec`) and JSON payloads.
+
+When you provide a YAML test spec, the command automatically translates test cases into Evaluation API calls and infers the agent name from the spec's `subjectName` field. This means you can use the same test spec with both `sf agent test run` and `sf agent test run-eval`.
+
+When you provide a JSON payload, it's sent directly to the API with optional normalization. The normalizer auto-corrects common field name mistakes, converts shorthand references to JSONPath, and injects defaults. Use `--no-normalize` to disable this auto-normalization.
+
+Supports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions, semantic similarity scoring, and LLM-based quality ratings.
+
+# flags.spec.summary
+
+Path to test spec file (YAML or JSON). Use `-` for stdin.
+
+# flags.api-name.summary
+
+Agent DeveloperName (also called API name) to resolve agent_id and agent_version_id. Auto-inferred from the YAML spec's subjectName.
+
+# flags.wait.summary
+
+Number of minutes to wait for results.
+
+# flags.result-format.summary
+
+Format of the agent test results.
+
+# flags.batch-size.summary
+
+Number of tests per API request (max 5).
+
+# flags.no-normalize.summary
+
+Disable auto-normalization of field names and shorthand references.
+
+# examples
+
+- Run tests using a YAML test spec on the org with alias "my-org":
+
+  <%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org
+
+- Run tests using a YAML spec with explicit agent name override; use your default org:
+
+  <%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --api-name My_Agent --target-org my-org
+
+- Run tests using a JSON payload:
+
+  <%= config.bin %> <%= command.id %> --spec tests/eval-payload.json --target-org my-org
+
+- Run tests and output results in JUnit format; useful for continuous integration and deployment (CI/CD):
+
+  <%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org --result-format junit
+
+- Pipe JSON payload from stdin:
+
+  $ echo '{"tests":[...]}' | <%= config.bin %> <%= command.id %> --spec - --target-org my-org
+
+# info.batchProgress
+
+Running batch %s of %s (%s tests)...
+
+# info.testComplete
+
+Test %s: %s.
+
+# info.summary
+
+Results: %s passed, %s failed, %s scored, %s errors.
+
+# info.yamlDetected
+
+Detected YAML test spec for agent '%s' with %s test case(s). Translating to Evaluation API format.
+
+# error.invalidPayload
+
+Invalid test payload: %s.
+
+# error.apiError
+
+Einstein Eval API error (HTTP %s): %s
+
+# error.agentNotFound
+
+No agent found with DeveloperName (also API name) '%s'. Verify that the agent exists in the target org.
+
+# error.agentVersionNotFound
+
+No published version found for agent '%s'. Make sure the agent has been published at least once.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@salesforce/plugin-agent",
   "description": "Commands to interact with Salesforce agents",
-  "version": "1.30.11",
+  "version": "1.31.3",
   "author": "Salesforce",
   "bugs": "https://github.com/forcedotcom/cli/issues",
   "enableO11y": true,
@@ -12,14 +12,14 @@
     "@inquirer/prompts": "^7.10.1",
     "@oclif/core": "^4",
     "@oclif/multi-stage-output": "^0.8.29",
-    "@salesforce/agents": "^0.23.4",
-    "@salesforce/core": "^8.26.2",
+    "@salesforce/agents": "^0.24.1",
+    "@salesforce/core": "^8.26.3",
     "@salesforce/kit": "^3.2.4",
     "@salesforce/sf-plugins-core": "^12.2.6",
     "@salesforce/source-deploy-retrieve": "^12.31.14",
     "@salesforce/types": "^1.6.0",
     "ansis": "^3.3.2",
-    "fast-xml-parser": "^5.4.1",
+    "fast-xml-parser": "^5.4.2",
     "glob": "^11.0.3",
     "ink": "5.0.1",
     "ink-text-input": "^6.0.0",
@@ -28,7 +28,7 @@
     "yaml": "^2.8.2"
   },
   "devDependencies": {
-    "@oclif/plugin-command-snapshot": "^5.3.10",
+    "@oclif/plugin-command-snapshot": "^5.3.11",
     "@oclif/test": "^4.1.15",
     "@salesforce/cli-plugins-testkit": "^5.3.41",
     "@salesforce/dev-scripts": "^11.0.4",