Skip to content

Commit 9974112

Browse files
shetzelshetzel
authored
Merge pull request #43 from salesforcecli/mdonnalley/tap-format
feat: add TAP option
2 parents 6ac93b6 + b9d7966 commit 9974112

7 files changed

Lines changed: 133 additions & 99 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"@inquirer/select": "^4.0.1",
1212
"@oclif/core": "^4",
1313
"@oclif/multi-stage-output": "^0.7.12",
14-
"@salesforce/agents": "^0.4.2",
14+
"@salesforce/agents": "^0.5.1",
1515
"@salesforce/core": "^8.8.0",
1616
"@salesforce/kit": "^3.2.1",
1717
"@salesforce/sf-plugins-core": "^12.1.0",

‎schemas/agent-test-results.json‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,27 @@
2020
"errorMessage": {
2121
"type": "string"
2222
},
23-
"testCases": {
24-
"type": "array",
25-
"items": {
26-
"$ref": "#/definitions/TestCaseResult"
27-
}
23+
"subjectName": {
24+
"type": "string"
25+
},
26+
"testSet": {
27+
"type": "object",
28+
"properties": {
29+
"name": {
30+
"type": "string"
31+
},
32+
"testCases": {
33+
"type": "array",
34+
"items": {
35+
"$ref": "#/definitions/TestCaseResult"
36+
}
37+
}
38+
},
39+
"required": ["name", "testCases"],
40+
"additionalProperties": false
2841
}
2942
},
30-
"required": ["status", "startTime", "testCases"],
43+
"required": ["status", "startTime", "subjectName", "testSet"],
3144
"additionalProperties": false
3245
},
3346
"TestStatus": {

‎src/flags.ts‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,7 @@ Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
1111
const messages = Messages.loadMessages('@salesforce/plugin-agent', 'shared');
1212

1313
export const resultFormatFlag = Flags.option({
14-
options: [
15-
'json',
16-
'human',
17-
'junit',
18-
// 'tap',
19-
] as const,
14+
options: ['json', 'human', 'junit', 'tap'] as const,
2015
default: 'human',
2116
summary: messages.getMessage('flags.result-format.summary'),
2217
});

‎src/handleTestResults.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
*/
77
import { join } from 'node:path';
88
import { writeFile, mkdir } from 'node:fs/promises';
9-
import { AgentTestDetailsResponse, jsonFormat, humanFormat, junitFormat } from '@salesforce/agents';
9+
import { AgentTestDetailsResponse, jsonFormat, humanFormat, junitFormat, tapFormat } from '@salesforce/agents';
1010
import { Ux } from '@salesforce/sf-plugins-core/Ux';
1111

1212
async function writeFileToDir(outputDir: string, fileName: string, content: string): Promise<void> {
@@ -24,7 +24,7 @@ export async function handleTestResults({
2424
outputDir,
2525
}: {
2626
id: string;
27-
format: 'human' | 'json' | 'junit';
27+
format: 'human' | 'json' | 'junit' | 'tap';
2828
results: AgentTestDetailsResponse | undefined;
2929
jsonEnabled: boolean;
3030
outputDir?: string;
@@ -59,4 +59,12 @@ export async function handleTestResults({
5959
await writeFileToDir(outputDir, `test-result-${id}.xml`, formatted);
6060
}
6161
}
62+
63+
if (format === 'tap') {
64+
const formatted = await tapFormat(results);
65+
ux.log(formatted);
66+
if (outputDir) {
67+
await writeFileToDir(outputDir, `test-result-${id}.txt`, formatted);
68+
}
69+
}
6270
}

‎test/commands/agent/test/results.nut.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ describe('agent test results NUTs', () => {
4747
).jsonOutput;
4848

4949
expect(output?.result.status).to.equal('COMPLETED');
50-
expect(output?.result.testCases.length).to.equal(2);
50+
expect(output?.result.testSet.testCases.length).to.equal(2);
5151

5252
// check that cache does not have an entry
5353
const cache = await AgentTestCache.create();

‎test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json‎

Lines changed: 92 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -3,80 +3,98 @@
33
"startTime": "2024-11-28T12:00:00Z",
44
"endTime": "2024-11-28T12:05:00Z",
55
"errorMessage": null,
6-
"testCases": [
7-
{
8-
"status": "COMPLETED",
9-
"number": 1,
10-
"startTime": "2024-11-28T12:00:10Z",
11-
"endTime": "2024-11-28T12:00:20Z",
12-
"generatedData": {
13-
"type": "AGENT",
14-
"actionsSequence": ["Action1", "Action2"],
15-
"outcome": "Success",
16-
"topic": "Mathematics",
17-
"inputTokensCount": 50,
18-
"outputTokensCount": 55
19-
},
20-
"expectationResults": [
21-
{
22-
"name": "topic_sequence_match",
23-
"actualValue": "Result A",
24-
"expectedValue": "Result A",
25-
"score": 1.0,
26-
"result": "Passed",
27-
"metricLabel": "Accuracy",
28-
"metricExplainability": "Measures the correctness of the result.",
29-
"status": "Completed",
30-
"startTime": "2024-11-28T12:00:12Z",
31-
"endTime": "2024-11-28T12:00:13Z",
32-
"errorCode": null,
33-
"errorMessage": null
6+
"subjectName": "Copilot_for_Salesforce",
7+
"testSet": {
8+
"name": "CRM_Sanity_v1",
9+
"testCases": [
10+
{
11+
"status": "COMPLETED",
12+
"number": 1,
13+
"startTime": "2024-11-28T12:00:10Z",
14+
"endTime": "2024-11-28T12:00:20Z",
15+
"generatedData": {
16+
"type": "AGENT",
17+
"actionsSequence": ["Action1", "Action2"],
18+
"outcome": "Success",
19+
"topic": "Mathematics",
20+
"inputTokensCount": 50,
21+
"outputTokensCount": 55
3422
},
35-
{
36-
"name": "action_sequence_match",
37-
"actualValue": "Result B",
38-
"expectedValue": "Result B",
39-
"score": 0.9,
40-
"result": "Passed",
41-
"metricLabel": "Precision",
42-
"metricExplainability": "Measures the precision of the result.",
43-
"status": "Completed",
44-
"startTime": "2024-11-28T12:00:14Z",
45-
"endTime": "2024-11-28T12:00:15Z",
46-
"errorCode": null,
47-
"errorMessage": null
48-
}
49-
]
50-
},
51-
{
52-
"status": "ERROR",
53-
"number": 2,
54-
"startTime": "2024-11-28T12:00:30Z",
55-
"endTime": "2024-11-28T12:00:40Z",
56-
"generatedData": {
57-
"type": "AGENT",
58-
"actionsSequence": ["Action3", "Action4"],
59-
"outcome": "Failure",
60-
"topic": "Physics",
61-
"inputTokensCount": 60,
62-
"outputTokensCount": 50
23+
"expectationResults": [
24+
{
25+
"name": "topic_sequence_match",
26+
"actualValue": "Result A",
27+
"expectedValue": "Result A",
28+
"score": 1.0,
29+
"result": "Passed",
30+
"metricLabel": "Accuracy",
31+
"metricExplainability": "Measures the correctness of the result.",
32+
"status": "Completed",
33+
"startTime": "2024-11-28T12:00:12Z",
34+
"endTime": "2024-11-28T12:00:13Z",
35+
"errorCode": null,
36+
"errorMessage": null
37+
},
38+
{
39+
"name": "action_sequence_match",
40+
"actualValue": "Result B",
41+
"expectedValue": "Result B",
42+
"score": 0.9,
43+
"result": "Passed",
44+
"metricLabel": "Precision",
45+
"metricExplainability": "Measures the precision of the result.",
46+
"status": "Completed",
47+
"startTime": "2024-11-28T12:00:14Z",
48+
"endTime": "2024-11-28T12:00:15Z",
49+
"errorCode": null,
50+
"errorMessage": null
51+
}
52+
]
6353
},
64-
"expectationResults": [
65-
{
66-
"name": "topic_sequence_match",
67-
"actualValue": "Result C",
68-
"expectedValue": "Result D",
69-
"score": 0.5,
70-
"result": "Failed",
71-
"metricLabel": "Accuracy",
72-
"metricExplainability": "Measures the correctness of the result.",
73-
"status": "Completed",
74-
"startTime": "2024-11-28T12:00:32Z",
75-
"endTime": "2024-11-28T12:00:33Z",
76-
"errorCode": null,
77-
"errorMessage": null
78-
}
79-
]
80-
}
81-
]
54+
{
55+
"status": "ERROR",
56+
"number": 2,
57+
"startTime": "2024-11-28T12:00:30Z",
58+
"endTime": "2024-11-28T12:00:40Z",
59+
"generatedData": {
60+
"type": "AGENT",
61+
"actionsSequence": ["Action3", "Action4"],
62+
"outcome": "Failure",
63+
"topic": "Physics",
64+
"inputTokensCount": 60,
65+
"outputTokensCount": 50
66+
},
67+
"expectationResults": [
68+
{
69+
"name": "topic_sequence_match",
70+
"actualValue": "Result C",
71+
"expectedValue": "Result D",
72+
"score": 0.5,
73+
"result": "Failed",
74+
"metricLabel": "Accuracy",
75+
"metricExplainability": "Measures the correctness of the result.",
76+
"status": "Completed",
77+
"startTime": "2024-11-28T12:00:32Z",
78+
"endTime": "2024-11-28T12:00:33Z",
79+
"errorCode": null,
80+
"errorMessage": "Expected \"Result D\" but got \"Result C\"."
81+
},
82+
{
83+
"name": "topic_sequence_match",
84+
"actualValue": "Result C",
85+
"expectedValue": "Result D",
86+
"score": 0.5,
87+
"result": "Failed",
88+
"metricLabel": "Accuracy",
89+
"metricExplainability": "Measures the correctness of the result.",
90+
"status": "Completed",
91+
"startTime": "2024-11-28T12:00:32Z",
92+
"endTime": "2024-11-28T12:00:33Z",
93+
"errorCode": null,
94+
"errorMessage": "Expected \"Result D\" but got \"Result C\"."
95+
}
96+
]
97+
}
98+
]
99+
}
82100
}

‎yarn.lock‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1422,16 +1422,16 @@
14221422
resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
14231423
integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
14241424

1425-
"@salesforce/agents@^0.4.2":
1426-
version "0.4.2"
1427-
resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.4.2.tgz#1caaf6a521cc3385a0411840876b77147aea880a"
1428-
integrity sha512-oTHEf3Dp6T8sYZzB/6V0HaZg7k/aZfTIFsoqYeewWpEa/KP0oopGHWTO9my15n1Guoq0qif6sWqX+RwWpqGgWw==
1425+
"@salesforce/agents@^0.5.1":
1426+
version "0.5.1"
1427+
resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.5.1.tgz#b6de16004505432c226c02f612c6b0b7b6227f6f"
1428+
integrity sha512-FGpCQ3PVzZunoaQVPAJG05eqafOvf2P7fx2w5aZYVg9yqwM/UnBpTBKVvkmdZDsBRTUYaExr6tvboaMc5Hsfzw==
14291429
dependencies:
14301430
"@oclif/table" "^0.3.5"
14311431
"@salesforce/core" "^8.8.0"
14321432
"@salesforce/kit" "^3.2.3"
14331433
"@salesforce/sf-plugins-core" "^12.1.0"
1434-
"@salesforce/source-deploy-retrieve" "^12.10.2"
1434+
"@salesforce/source-deploy-retrieve" "^12.10.3"
14351435
fast-xml-parser "^4"
14361436
nock "^13.5.6"
14371437

@@ -1577,10 +1577,10 @@
15771577
cli-progress "^3.12.0"
15781578
terminal-link "^3.0.0"
15791579

1580-
"@salesforce/source-deploy-retrieve@^12.10.2":
1581-
version "12.10.2"
1582-
resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.10.2.tgz#c3737f3751f84cb4754b666edd83c014c91b87bb"
1583-
integrity sha512-V7V+nCxhJBcSEh5gJIZt44bsnYJ53HXt3Dec4gphu/Z8mu6Rr3b3yDC3Zw2EKyiDwYYxWvp0pjhqcO2xOTnNLw==
1580+
"@salesforce/source-deploy-retrieve@^12.10.3":
1581+
version "12.10.3"
1582+
resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.10.3.tgz#fa16910781188877ffdfa5fde3a0318c0dfe3d07"
1583+
integrity sha512-bKIcN6VJajre2chF1xhPCjtR9gZpp8PrFFZ55UcWUMkoFAXscBPRJ7poAeorted3qMzS6wx+AuB27qYUCO+4iQ==
15841584
dependencies:
15851585
"@salesforce/core" "^8.8.0"
15861586
"@salesforce/kit" "^3.2.2"

0 commit comments

Comments
 (0)