Skip to content

Commit 8bc7934

Browse files
RidaZubairRidaZubair
authored
test(anaconda-ai): add e2e CLI tests for anaconda ai server lifecycle and negative cases (#114)
* cover sever testcases * cover sever testcases * cover sever testcases * Address feedback comment
1 parent d985cb2 commit 8bc7934

4 files changed

Lines changed: 198 additions & 18 deletions

File tree

‎tests/e2e/pages/cli/anaconda-ai-cmds.ts‎

Lines changed: 125 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,31 @@ export class AnacondaAiCli {
4242
this.assertModelResponseData(models);
4343
}
4444

45+
public async runAnacondaAiServersListCommand(): Promise<ShellResult> {
46+
return await shellCommand(cliCommands.anacondaAiServersListCmd);
47+
}
48+
49+
public verifyAnacondaAiServersListCommand(result: ShellResult): void {
50+
verifyShellExitCode(result, 'anaconda ai servers --json');
51+
52+
const servers = JSON.parse(stripAnsiSgrAndTrim(result.output)) as ServerApi[];
53+
expect(Array.isArray(servers), 'servers output should be an array').toBe(true);
54+
}
55+
56+
// Verifies that a server for the given model appears in the servers list with status "running"
57+
public verifyRunningServerInList(result: ShellResult, modelName: string, modelQuantization: string): void {
58+
verifyShellExitCode(result, 'anaconda ai servers --json');
59+
60+
const servers = JSON.parse(stripAnsiSgrAndTrim(result.output)) as ServerApi[];
61+
const expectedModelFile = `${modelName}_${modelQuantization}.gguf`;
62+
const server = servers.find((s) => s.model === expectedModelFile);
63+
64+
expect(server, `Expected server for model "${expectedModelFile}" to appear in servers list`).toBeDefined();
65+
expect(server!.server_id, `Expected server_id to contain model name "${modelName}"`).toContain(modelName);
66+
expect(server!.model, `Expected model to be "${expectedModelFile}"`).toBe(expectedModelFile);
67+
expect(server!.status, `Expected server status to be "running"`).toBe('running');
68+
}
69+
4570
// Executes `anaconda ai download <model>/<quant>` (positional; no --model)
4671
public async runDownloadModelCommand(modelName: string, modelQuantization: string): Promise<ShellResult> {
4772
return await shellCommand(cliCommands.downloadModelCmd(modelName, modelQuantization));
@@ -61,15 +86,110 @@ export class AnacondaAiCli {
6186
expect(
6287
isDownloadedNow || isAlreadyDownloaded,
6388
'Expected the model to be either newly downloaded or already downloaded',
64-
)
65-
.toBeTruthy();
89+
).toBeTruthy();
6690
}
6791

6892
public verifyInvalidDownloadModelCommand(result: ShellResult): void {
69-
expect(result.exitCode, `Expected invalid download command to fail, but got exit code ${result.exitCode}`,).not.toBe(0);
93+
expect(
94+
result.exitCode,
95+
`Expected invalid download command to fail, but got exit code ${result.exitCode}`,
96+
).not.toBe(0);
7097
const output = stripAnsiSgrAndTrim(result.output).toLowerCase();
7198
expect(output, 'Expected the model to be invalid').toContain('error');
72-
expect(output, 'Expected the error message to be invalid').toContain(INVALID_MODEL_ERROR_MESSAGE);
99+
expect(output, 'Expected output should contain invalid model error message').toContain(INVALID_MODEL_ERROR_MESSAGE);
100+
}
101+
102+
// Executes `anaconda ai launch <model>/<quant> --detach`
103+
public async runLaunchModelCommand(modelName: string, modelQuantization: string): Promise<ShellResult> {
104+
return await shellCommand(cliCommands.launchModelCmd(modelName, modelQuantization));
105+
}
106+
107+
// Validates that launching a model that already has a running server fails with AnacondaAIException
108+
public verifyDuplicateLaunchModelCommand(result: ShellResult): void {
109+
expect(result.exitCode, 'Expected non-zero exit code for duplicate launch').not.toBe(0);
110+
111+
const output = stripAnsiSgrAndTrim(result.output).toLowerCase();
112+
expect(
113+
output.includes('anacondaaiexception') || output.includes('already exists'),
114+
'Expected AnacondaAIException about duplicate server',
115+
).toBeTruthy();
116+
}
117+
118+
// Validates model server launched successfully
119+
public verifyLaunchModelCommand(result: ShellResult, modelName: string, modelQuantization: string): void {
120+
verifyShellExitCode(result, 'anaconda ai launch <model>/<quant>');
121+
122+
const output = stripAnsiSgrAndTrim(result.output).toLowerCase();
123+
const expectedModelFile = `${modelName}_${modelQuantization}.gguf`.toLowerCase();
124+
expect(output.includes('running'), 'Expected status to be "running"').toBeTruthy();
125+
expect(
126+
output.includes('inference/serve/'),
127+
'Expected "inference/serve/" is running',
128+
).toBeTruthy();
129+
expect(
130+
output.includes(expectedModelFile),
131+
`Expected model "${expectedModelFile}" to appear in output`,
132+
).toBeTruthy();
133+
}
134+
135+
// Executes `anaconda ai stop <modelName>_<modelQuantization>.gguf`
136+
public async runStopModelCommand(modelName: string, modelQuantization: string): Promise<ShellResult> {
137+
return await shellCommand(cliCommands.stopModelCmd(modelName, modelQuantization));
138+
}
139+
140+
// Validates model server stopped successfully
141+
public verifyStopModelCommand(result: ShellResult, modelName: string, modelQuantization: string): void {
142+
verifyShellExitCode(result, 'anaconda ai stop <model>.gguf');
143+
144+
const output = stripAnsiSgrAndTrim(result.output).toLowerCase();
145+
const expectedModelFile = `${modelName}_${modelQuantization}.gguf`.toLowerCase();
146+
expect(output.includes('stopped'), 'Expected status to be "stopped"').toBeTruthy();
147+
expect(output.includes('success'), 'Expected "Success" in output').toBeTruthy();
148+
expect(
149+
output.includes(expectedModelFile),
150+
`Expected model "${expectedModelFile}" to appear in output`,
151+
).toBeTruthy();
152+
}
153+
154+
// Executes `anaconda ai stop <modelName>_<modelQuantization>.gguf --rm`
155+
public async runStopAndRemoveModelCommand(modelName: string, modelQuantization: string): Promise<ShellResult> {
156+
return await shellCommand(cliCommands.stopAndRemoveModelCmd(modelName, modelQuantization));
157+
}
158+
159+
// Validates model server deleted successfully
160+
public verifyStopAndRemoveModelCommand(result: ShellResult, modelName: string, modelQuantization: string): void {
161+
verifyShellExitCode(result, 'anaconda ai stop <model>.gguf --rm');
162+
163+
const output = stripAnsiSgrAndTrim(result.output).toLowerCase();
164+
const expectedModelFile = `${modelName}_${modelQuantization}.gguf`.toLowerCase();
165+
expect(output.includes('deleted'), 'Expected status to be "deleted"').toBeTruthy();
166+
expect(output.includes('success'), 'Expected "Success" in output').toBeTruthy();
167+
expect(
168+
output.includes(expectedModelFile),
169+
`Expected model "${expectedModelFile}" to appear in output`,
170+
).toBeTruthy();
171+
}
172+
173+
// Validates that launching with an invalid format exits non-zero with a ValueError
174+
public verifyLaunchModelInvalidFormatCommand(result: ShellResult): void {
175+
expect(result.exitCode, 'Expected non-zero exit code for invalid format').not.toBe(0);
176+
177+
const output = stripAnsiSgrAndTrim(result.output).toLowerCase();
178+
expect(
179+
output.includes('valueerror') && output.includes('does not look like a quantized model name'),
180+
'Expected ValueError about invalid model name format',
181+
).toBeTruthy();
182+
}
183+
184+
// Validates that launching an unknown model exits non-zero with a ModelNotFound error
185+
public verifyLaunchModelNotFoundCommand(result: ShellResult): void {
186+
expect(result.exitCode, 'Expected non-zero exit code for unknown model').not.toBe(0);
187+
188+
const output = stripAnsiSgrAndTrim(result.output).toLowerCase();
189+
expect(
190+
output.includes('modelnotfound') || output.includes('was not found'),
191+
'Expected ModelNotFound error for unknown model',
192+
).toBeTruthy();
73193
}
74194

75195
private assertModelResponseData(models: ModelApi[]): void {
@@ -89,15 +209,4 @@ export class AnacondaAiCli {
89209
});
90210
});
91211
}
92-
93-
public async runAnacondaAiServersListCommand(): Promise<ShellResult> {
94-
return await shellCommand(cliCommands.anacondaAiServersListCmd);
95-
}
96-
97-
public verifyAnacondaAiServersListCommand(result: ShellResult): void {
98-
verifyShellExitCode(result, 'anaconda ai servers --json');
99-
100-
const servers = JSON.parse(stripAnsiSgrAndTrim(result.output)) as ServerApi[];
101-
expect(Array.isArray(servers), 'servers output should be an array').toBe(true);
102-
}
103-
}
212+
}

‎tests/e2e/pages/cli/cliCommands.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,15 @@ export const downloadModelCmd = (modelName: string, modelQuantization: string):
4242

4343
// Anaconda AI Servers Command
4444
export const anacondaAiServersListCmd = condaRun('anaconda ai servers --json');
45+
46+
// Launch server for model command
47+
export const launchModelCmd = (modelName: string, modelQuantization: string): string =>
48+
condaRun(`anaconda ai launch ${modelName}/${modelQuantization} --detach`);
49+
50+
// Stop server for model command — takes the .gguf filename: <modelName>_<modelQuantization>.gguf
51+
export const stopModelCmd = (modelName: string, modelQuantization: string): string =>
52+
condaRun(`anaconda ai stop ${modelName}_${modelQuantization}.gguf`);
53+
54+
// Stop and remove server — same as stop but with --rm to delete it
55+
export const stopAndRemoveModelCmd = (modelName: string, modelQuantization: string): string =>
56+
condaRun(`anaconda ai stop ${modelName}_${modelQuantization}.gguf --rm`);

‎tests/e2e/specs/cli/anaconda-ai.spec.ts‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,17 @@
11
import { test } from '@fixture';
2-
import { DOWNLOAD_TEST_MODEL_NAME, DOWNLOAD_TEST_MODEL_QUANTIZATION, INVALID_MODEL_NAME, INVALID_MODEL_QUANTIZATION } from '@testdata/model-api';
2+
import {
3+
DOWNLOAD_TEST_MODEL_NAME,
4+
DOWNLOAD_TEST_MODEL_QUANTIZATION,
5+
INVALID_MODEL_NAME,
6+
INVALID_MODEL_QUANTIZATION,
7+
} from '@testdata/model-api';
38

49
test.describe('Anaconda AI CLI Commands @anaconda-ai', () => {
510
test('anaconda ai --help', async ({ anacondaAiCli }) => {
611
const result = await anacondaAiCli.runAnacondaAiHelpCommand();
712
anacondaAiCli.verifyAnacondaAiHelpCommand(result);
813
});
14+
915
test('anaconda ai models list command', async ({ anacondaAiCli }) => {
1016
const result = await anacondaAiCli.runAnacondaAiModelsListCommand();
1117
anacondaAiCli.verifyAnacondaAiModelsListCommand(result);
@@ -37,4 +43,58 @@ test.describe('Anaconda AI CLI Commands @anaconda-ai', () => {
3743
anacondaAiCli.verifyAnacondaAiServersListCommand(result);
3844
});
3945

46+
test('anaconda ai server lifecycle: launch, verify, stop, and delete a model server', async ({ anacondaAiCli }) => {
47+
await test.step('step 1: launch model server and verify it is running', async () => {
48+
const launchResult = await anacondaAiCli.runLaunchModelCommand(
49+
DOWNLOAD_TEST_MODEL_NAME,
50+
DOWNLOAD_TEST_MODEL_QUANTIZATION,
51+
);
52+
anacondaAiCli.verifyLaunchModelCommand(launchResult, DOWNLOAD_TEST_MODEL_NAME, DOWNLOAD_TEST_MODEL_QUANTIZATION);
53+
});
54+
55+
await test.step('step 2: verify server appears in servers list with status "running"', async () => {
56+
const serversResult = await anacondaAiCli.runAnacondaAiServersListCommand();
57+
anacondaAiCli.verifyRunningServerInList(serversResult, DOWNLOAD_TEST_MODEL_NAME, DOWNLOAD_TEST_MODEL_QUANTIZATION);
58+
});
59+
60+
await test.step('step 3: launching the same server again returns AnacondaAIException', async () => {
61+
const duplicateResult = await anacondaAiCli.runLaunchModelCommand(
62+
DOWNLOAD_TEST_MODEL_NAME,
63+
DOWNLOAD_TEST_MODEL_QUANTIZATION,
64+
);
65+
anacondaAiCli.verifyDuplicateLaunchModelCommand(duplicateResult);
66+
});
67+
68+
await test.step('step 4: stop the model server and verify it is stopped', async () => {
69+
const stopResult = await anacondaAiCli.runStopModelCommand(
70+
DOWNLOAD_TEST_MODEL_NAME,
71+
DOWNLOAD_TEST_MODEL_QUANTIZATION,
72+
);
73+
anacondaAiCli.verifyStopModelCommand(stopResult, DOWNLOAD_TEST_MODEL_NAME, DOWNLOAD_TEST_MODEL_QUANTIZATION);
74+
});
75+
76+
await test.step('step 5: delete the model server and verify it is removed', async () => {
77+
const deleteResult = await anacondaAiCli.runStopAndRemoveModelCommand(
78+
DOWNLOAD_TEST_MODEL_NAME,
79+
DOWNLOAD_TEST_MODEL_QUANTIZATION,
80+
);
81+
anacondaAiCli.verifyStopAndRemoveModelCommand(deleteResult, DOWNLOAD_TEST_MODEL_NAME, DOWNLOAD_TEST_MODEL_QUANTIZATION);
82+
});
83+
});
84+
85+
test('anaconda ai launch - invalid format returns ValueError', async ({ anacondaAiCli }) => {
86+
const result = await anacondaAiCli.runLaunchModelCommand(
87+
INVALID_MODEL_NAME,
88+
INVALID_MODEL_QUANTIZATION,
89+
);
90+
anacondaAiCli.verifyLaunchModelInvalidFormatCommand(result);
91+
});
92+
93+
test('anaconda ai launch - unknown model returns ModelNotFound', async ({ anacondaAiCli }) => {
94+
const result = await anacondaAiCli.runLaunchModelCommand(
95+
INVALID_MODEL_NAME,
96+
DOWNLOAD_TEST_MODEL_QUANTIZATION,
97+
);
98+
anacondaAiCli.verifyLaunchModelNotFoundCommand(result);
99+
});
40100
});

‎tests/e2e/testdata/model-api.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,3 @@ export const DOWNLOAD_TEST_MODEL_QUANTIZATION = 'q4_k_m';
2929
export const INVALID_MODEL_NAME = 'invalid-model';
3030
export const INVALID_MODEL_QUANTIZATION = 'invalid-quantization';
3131
export const INVALID_MODEL_ERROR_MESSAGE = 'you must include the quantization method in the model';
32-

0 commit comments

Comments
 (0)