Skip to content

Commit b9d7966

Browse files
mdonnalleymdonnalley
committed
chore: bump agents
1 parent 0266d73 commit b9d7966

5 files changed

Lines changed: 113 additions & 90 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
"@inquirer/select": "^4.0.1",
1111
"@oclif/core": "^4",
1212
"@oclif/multi-stage-output": "^0.7.12",
13-
"@salesforce/agents": "^0.5.0",
13+
"@salesforce/agents": "^0.5.1",
1414
"@salesforce/core": "^8.8.0",
1515
"@salesforce/kit": "^3.2.1",
1616
"@salesforce/sf-plugins-core": "^12.1.0",

‎schemas/agent-test-results.json‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,24 @@
2323
"subjectName": {
2424
"type": "string"
2525
},
26-
"testSetName": {
27-
"type": "string"
28-
},
29-
"testCases": {
30-
"type": "array",
31-
"items": {
32-
"$ref": "#/definitions/TestCaseResult"
33-
}
26+
"testSet": {
27+
"type": "object",
28+
"properties": {
29+
"name": {
30+
"type": "string"
31+
},
32+
"testCases": {
33+
"type": "array",
34+
"items": {
35+
"$ref": "#/definitions/TestCaseResult"
36+
}
37+
}
38+
},
39+
"required": ["name", "testCases"],
40+
"additionalProperties": false
3441
}
3542
},
36-
"required": ["status", "startTime", "subjectName", "testSetName", "testCases"],
43+
"required": ["status", "startTime", "subjectName", "testSet"],
3744
"additionalProperties": false
3845
},
3946
"TestStatus": {

‎test/commands/agent/test/results.nut.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ describe('agent test results NUTs', () => {
4747
).jsonOutput;
4848

4949
expect(output?.result.status).to.equal('COMPLETED');
50-
expect(output?.result.testCases.length).to.equal(2);
50+
expect(output?.result.testSet.testCases.length).to.equal(2);
5151

5252
// check that cache does not have an entry
5353
const cache = await AgentTestCache.create();

‎test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json‎

Lines changed: 91 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -4,81 +4,97 @@
44
"endTime": "2024-11-28T12:05:00Z",
55
"errorMessage": null,
66
"subjectName": "Copilot_for_Salesforce",
7-
"testSetName": "CRM_Sanity_v1",
8-
"testCases": [
9-
{
10-
"status": "COMPLETED",
11-
"number": 1,
12-
"startTime": "2024-11-28T12:00:10Z",
13-
"endTime": "2024-11-28T12:00:20Z",
14-
"generatedData": {
15-
"type": "AGENT",
16-
"actionsSequence": ["Action1", "Action2"],
17-
"outcome": "Success",
18-
"topic": "Mathematics",
19-
"inputTokensCount": 50,
20-
"outputTokensCount": 55
21-
},
22-
"expectationResults": [
23-
{
24-
"name": "topic_sequence_match",
25-
"actualValue": "Result A",
26-
"expectedValue": "Result A",
27-
"score": 1.0,
28-
"result": "Passed",
29-
"metricLabel": "Accuracy",
30-
"metricExplainability": "Measures the correctness of the result.",
31-
"status": "Completed",
32-
"startTime": "2024-11-28T12:00:12Z",
33-
"endTime": "2024-11-28T12:00:13Z",
34-
"errorCode": null,
35-
"errorMessage": null
7+
"testSet": {
8+
"name": "CRM_Sanity_v1",
9+
"testCases": [
10+
{
11+
"status": "COMPLETED",
12+
"number": 1,
13+
"startTime": "2024-11-28T12:00:10Z",
14+
"endTime": "2024-11-28T12:00:20Z",
15+
"generatedData": {
16+
"type": "AGENT",
17+
"actionsSequence": ["Action1", "Action2"],
18+
"outcome": "Success",
19+
"topic": "Mathematics",
20+
"inputTokensCount": 50,
21+
"outputTokensCount": 55
3622
},
37-
{
38-
"name": "action_sequence_match",
39-
"actualValue": "Result B",
40-
"expectedValue": "Result B",
41-
"score": 0.9,
42-
"result": "Passed",
43-
"metricLabel": "Precision",
44-
"metricExplainability": "Measures the precision of the result.",
45-
"status": "Completed",
46-
"startTime": "2024-11-28T12:00:14Z",
47-
"endTime": "2024-11-28T12:00:15Z",
48-
"errorCode": null,
49-
"errorMessage": null
50-
}
51-
]
52-
},
53-
{
54-
"status": "ERROR",
55-
"number": 2,
56-
"startTime": "2024-11-28T12:00:30Z",
57-
"endTime": "2024-11-28T12:00:40Z",
58-
"generatedData": {
59-
"type": "AGENT",
60-
"actionsSequence": ["Action3", "Action4"],
61-
"outcome": "Failure",
62-
"topic": "Physics",
63-
"inputTokensCount": 60,
64-
"outputTokensCount": 50
23+
"expectationResults": [
24+
{
25+
"name": "topic_sequence_match",
26+
"actualValue": "Result A",
27+
"expectedValue": "Result A",
28+
"score": 1.0,
29+
"result": "Passed",
30+
"metricLabel": "Accuracy",
31+
"metricExplainability": "Measures the correctness of the result.",
32+
"status": "Completed",
33+
"startTime": "2024-11-28T12:00:12Z",
34+
"endTime": "2024-11-28T12:00:13Z",
35+
"errorCode": null,
36+
"errorMessage": null
37+
},
38+
{
39+
"name": "action_sequence_match",
40+
"actualValue": "Result B",
41+
"expectedValue": "Result B",
42+
"score": 0.9,
43+
"result": "Passed",
44+
"metricLabel": "Precision",
45+
"metricExplainability": "Measures the precision of the result.",
46+
"status": "Completed",
47+
"startTime": "2024-11-28T12:00:14Z",
48+
"endTime": "2024-11-28T12:00:15Z",
49+
"errorCode": null,
50+
"errorMessage": null
51+
}
52+
]
6553
},
66-
"expectationResults": [
67-
{
68-
"name": "topic_sequence_match",
69-
"actualValue": "Result C",
70-
"expectedValue": "Result D",
71-
"score": 0.5,
72-
"result": "Failed",
73-
"metricLabel": "Accuracy",
74-
"metricExplainability": "Measures the correctness of the result.",
75-
"status": "Completed",
76-
"startTime": "2024-11-28T12:00:32Z",
77-
"endTime": "2024-11-28T12:00:33Z",
78-
"errorCode": null,
79-
"errorMessage": null
80-
}
81-
]
82-
}
83-
]
54+
{
55+
"status": "ERROR",
56+
"number": 2,
57+
"startTime": "2024-11-28T12:00:30Z",
58+
"endTime": "2024-11-28T12:00:40Z",
59+
"generatedData": {
60+
"type": "AGENT",
61+
"actionsSequence": ["Action3", "Action4"],
62+
"outcome": "Failure",
63+
"topic": "Physics",
64+
"inputTokensCount": 60,
65+
"outputTokensCount": 50
66+
},
67+
"expectationResults": [
68+
{
69+
"name": "topic_sequence_match",
70+
"actualValue": "Result C",
71+
"expectedValue": "Result D",
72+
"score": 0.5,
73+
"result": "Failed",
74+
"metricLabel": "Accuracy",
75+
"metricExplainability": "Measures the correctness of the result.",
76+
"status": "Completed",
77+
"startTime": "2024-11-28T12:00:32Z",
78+
"endTime": "2024-11-28T12:00:33Z",
79+
"errorCode": null,
80+
"errorMessage": "Expected \"Result D\" but got \"Result C\"."
81+
},
82+
{
83+
"name": "topic_sequence_match",
84+
"actualValue": "Result C",
85+
"expectedValue": "Result D",
86+
"score": 0.5,
87+
"result": "Failed",
88+
"metricLabel": "Accuracy",
89+
"metricExplainability": "Measures the correctness of the result.",
90+
"status": "Completed",
91+
"startTime": "2024-11-28T12:00:32Z",
92+
"endTime": "2024-11-28T12:00:33Z",
93+
"errorCode": null,
94+
"errorMessage": "Expected \"Result D\" but got \"Result C\"."
95+
}
96+
]
97+
}
98+
]
99+
}
84100
}

‎yarn.lock‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1389,10 +1389,10 @@
13891389
resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
13901390
integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
13911391

1392-
"@salesforce/agents@^0.5.0":
1393-
version "0.5.0"
1394-
resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.5.0.tgz#f6e4106e4796dde6bc1c5d4045511fd7d27a4640"
1395-
integrity sha512-xmPCC3yOXFgsG0Mrt+BDRBVibJzHzBHlKws7szEeNY5it9g2rt0Knl/KzZZTDz9hGAkNCd94T4luCt653l7Pbg==
1392+
"@salesforce/agents@^0.5.1":
1393+
version "0.5.1"
1394+
resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.5.1.tgz#b6de16004505432c226c02f612c6b0b7b6227f6f"
1395+
integrity sha512-FGpCQ3PVzZunoaQVPAJG05eqafOvf2P7fx2w5aZYVg9yqwM/UnBpTBKVvkmdZDsBRTUYaExr6tvboaMc5Hsfzw==
13961396
dependencies:
13971397
"@oclif/table" "^0.3.5"
13981398
"@salesforce/core" "^8.8.0"

0 commit comments

Comments
 (0)