Skip to content

Commit f75a977

Browse files
katieperry4katieperry4
authored
PVC Serving E2E Test (opendatahub-io#4494)
* Add E2E test for deploying from PVC * pr fixes * remove feature flag flip
1 parent 250e496 commit f75a977

File tree

7 files changed

+259
-0
lines changed

7 files changed

+259
-0
lines changed

‎frontend/src/__tests__/cypress/cypress/fixtures/e2e/dataScienceProjects/testModelPvcDeployment.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# testModelPvcDeployment.cy.ts Test Data #
2+
projectResourceName: 'test-model-pvc-deployment'
3+
projectDisplayName: 'Test Model PVC Deployment'
4+
singleModelName: 'test-model'
5+
modelOpenVinoExamplePath: 'kserve-openvino-test/openvino-example-model'
6+
pvStorageName: 'model-pvc-e2e'

‎frontend/src/__tests__/cypress/cypress/fixtures/resources/yaml/pvc-loader-pod.yaml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: {{POD_NAME}}
5+
namespace: {{NAMESPACE}}
6+
spec:
7+
containers:
8+
- name: s3-downloader
9+
image: amazon/aws-cli
10+
command: ["/bin/bash", "-c"]
11+
args:
12+
- |
13+
set -e
14+
echo "Starting S3 copy from s3://{{AWS_S3_BUCKET}}/{{MODEL_PATH}} to /mnt/models/"
15+
16+
MAX_RETRIES=3
17+
COUNT=0
18+
19+
while [ $COUNT -lt $MAX_RETRIES ]; do
20+
echo "Attempt $((COUNT + 1)) of $MAX_RETRIES..."
21+
if aws s3 cp s3://{{AWS_S3_BUCKET}}/{{MODEL_PATH}} /mnt/models/ --region={{AWS_DEFAULT_REGION}} --recursive; then
22+
echo "S3 copy completed successfully"
23+
touch /mnt/models/.copy-complete
24+
exit 0
25+
fi
26+
echo "S3 copy failed. Retrying in 5 seconds..."
27+
COUNT=$((COUNT + 1))
28+
sleep 5
29+
done
30+
31+
echo "S3 copy failed after $MAX_RETRIES attempts"
32+
exit 1
33+
volumeMounts:
34+
- name: model-store
35+
mountPath: /mnt/models
36+
env:
37+
- name: AWS_ACCESS_KEY_ID
38+
value: "{{AWS_ACCESS_KEY_ID}}"
39+
- name: AWS_SECRET_ACCESS_KEY
40+
value: "{{AWS_SECRET_ACCESS_KEY}}"
41+
- name: AWS_DEFAULT_REGION
42+
value: "{{AWS_DEFAULT_REGION}}"
43+
- name: AWS_S3_ENDPOINT
44+
value: "{{AWS_S3_ENDPOINT}}"
45+
- name: AWS_S3_BUCKET
46+
value: "{{AWS_S3_BUCKET}}"
47+
volumes:
48+
- name: model-store
49+
persistentVolumeClaim:
50+
claimName: {{PVC_NAME}}
51+
restartPolicy: Never

‎frontend/src/__tests__/cypress/cypress/pages/modelServing.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,10 @@ class InferenceServiceModal extends ServingModal {
227227
return this.find().findByTestId('new-connection-radio');
228228
}
229229

230+
findExistingPVCConnectionOption() {
231+
return this.find().findByTestId('pvc-serving-radio');
232+
}
233+
230234
findExistingConnectionOption() {
231235
return this.find().findByTestId('existing-connection-radio');
232236
}
@@ -346,6 +350,10 @@ class InferenceServiceModal extends ServingModal {
346350
return this.find().findByTestId('folder-path');
347351
}
348352

353+
findUriLocationPathInput() {
354+
return this.find().findByTestId('field URI');
355+
}
356+
349357
findLocationPathInputError() {
350358
return this.find().findByTestId('folder-path-error');
351359
}

‎frontend/src/__tests__/cypress/cypress/tests/e2e/dataScienceProjects/models/testModelPvcDeployment.cy.ts‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
import {
2+
inferenceServiceModal,
3+
modelServingGlobal,
4+
} from '#~/__tests__/cypress/cypress/pages/modelServing';
5+
import { AWS_BUCKETS } from '#~/__tests__/cypress/cypress/utils/s3Buckets';
6+
import {
7+
checkInferenceServiceState,
8+
provisionProjectForModelServing,
9+
verifyS3CopyCompleted,
10+
} from '#~/__tests__/cypress/cypress/utils/oc_commands/modelServing';
11+
import { deleteOpenShiftProject } from '#~/__tests__/cypress/cypress/utils/oc_commands/project';
12+
import { loadDSPFixture } from '#~/__tests__/cypress/cypress/utils/dataLoader';
13+
import { HTPASSWD_CLUSTER_ADMIN_USER } from '#~/__tests__/cypress/cypress/utils/e2eUsers';
14+
import { retryableBefore } from '#~/__tests__/cypress/cypress/utils/retryableHooks';
15+
import { projectListPage, projectDetails } from '#~/__tests__/cypress/cypress/pages/projects';
16+
import { generateTestUUID } from '#~/__tests__/cypress/cypress/utils/uuidGenerator';
17+
import type {
18+
DataScienceProjectData,
19+
PVCLoaderPodReplacements,
20+
} from '#~/__tests__/cypress/cypress/types';
21+
import {
22+
clusterStorage,
23+
addClusterStorageModal,
24+
} from '#~/__tests__/cypress/cypress/pages/clusterStorage';
25+
import { createS3LoaderPod } from '#~/__tests__/cypress/cypress/utils/oc_commands/pvcLoaderPod';
26+
import { waitForPodReady } from '#~/__tests__/cypress/cypress/utils/oc_commands/baseCommands';
27+
28+
let testData: DataScienceProjectData;
29+
let projectName: string;
30+
let modelName: string;
31+
let modelFilePath: string;
32+
let pvStorageName: string;
33+
const awsBucket = 'BUCKET_1' as const;
34+
const awsAccessKeyId = AWS_BUCKETS.AWS_ACCESS_KEY_ID;
35+
const awsSecretAccessKey = AWS_BUCKETS.AWS_SECRET_ACCESS_KEY;
36+
const awsBucketName = AWS_BUCKETS.BUCKET_1.NAME;
37+
const awsBucketEndpoint = AWS_BUCKETS.BUCKET_1.ENDPOINT;
38+
const awsBucketRegion = AWS_BUCKETS.BUCKET_1.REGION;
39+
const podName = 'pvc-loader-pod';
40+
const uuid = generateTestUUID();
41+
42+
describe('Verify a model can be deployed from a PVC', () => {
43+
retryableBefore(() => {
44+
Cypress.on('uncaught:exception', (err) => {
45+
if (err.message.includes('Error: secrets "ds-pipeline-config" already exists')) {
46+
return false;
47+
}
48+
return true;
49+
});
50+
return loadDSPFixture('e2e/dataScienceProjects/testModelPvcDeployment.yaml').then(
51+
(fixtureData: DataScienceProjectData) => {
52+
testData = fixtureData;
53+
projectName = `${testData.projectResourceName}-${uuid}`;
54+
modelName = testData.singleModelName;
55+
modelFilePath = testData.modelOpenVinoExamplePath;
56+
pvStorageName = testData.pvStorageName;
57+
58+
if (!projectName) {
59+
throw new Error('Project name is undefined or empty in the loaded fixture');
60+
}
61+
// Create a Project for pipelines
62+
provisionProjectForModelServing(projectName, awsBucket);
63+
},
64+
);
65+
});
66+
after(() => {
67+
// Delete provisioned Project
68+
deleteOpenShiftProject(projectName, { wait: false, ignoreNotFound: true });
69+
});
70+
it(
71+
'should deploy a model from a PVC',
72+
{ tags: ['@Smoke', '@SmokeSet3', '@Dashboard', '@ModelServing'] },
73+
() => {
74+
cy.step('log into application with ${HTPASSWD_CLUSTER_ADMIN_USER.USERNAME}');
75+
cy.visitWithLogin('/', HTPASSWD_CLUSTER_ADMIN_USER);
76+
77+
// Navigate to the project
78+
cy.step('Navigate to the project');
79+
projectListPage.visit();
80+
projectListPage.filterProjectByName(projectName);
81+
projectListPage.findProjectLink(projectName).click();
82+
83+
// Navigate to cluster storage page
84+
cy.step('Navigate to cluster storage page');
85+
projectDetails.findSectionTab('cluster-storages').click();
86+
clusterStorage.findCreateButton().click();
87+
88+
// Enter cluster storage details
89+
cy.step('Enter cluster storage details');
90+
addClusterStorageModal.findNameInput().type(pvStorageName);
91+
92+
addClusterStorageModal.findModelStorageRadio().click();
93+
addClusterStorageModal.findModelPathInput().type(modelFilePath);
94+
addClusterStorageModal.findModelNameInput().type(modelName);
95+
96+
addClusterStorageModal.findSubmitButton().click({ force: true });
97+
98+
// Verify the cluster storage is created
99+
const pvcRow = clusterStorage.getClusterStorageRow(pvStorageName);
100+
pvcRow.find().should('exist');
101+
pvcRow.findStorageTypeColumn().should('contain', 'Model storage');
102+
103+
pvcRow.findConnectedResources().should('contain', modelName);
104+
105+
const pvcReplacements: PVCLoaderPodReplacements = {
106+
NAMESPACE: projectName,
107+
PVC_NAME: pvStorageName,
108+
AWS_S3_BUCKET: awsBucketName,
109+
AWS_S3_ENDPOINT: awsBucketEndpoint,
110+
AWS_ACCESS_KEY_ID: awsAccessKeyId,
111+
AWS_SECRET_ACCESS_KEY: awsSecretAccessKey,
112+
AWS_DEFAULT_REGION: awsBucketRegion,
113+
POD_NAME: podName,
114+
MODEL_PATH: modelFilePath,
115+
};
116+
117+
// Create pod to mount the PVC
118+
cy.step('Create pod to mount the PVC');
119+
createS3LoaderPod(pvcReplacements);
120+
121+
// Verify the pod is ready
122+
cy.step('Verify the pod is ready');
123+
waitForPodReady(podName, projectName);
124+
125+
// Verify the S3 copy completed successfully
126+
cy.step('Verify S3 copy completed');
127+
verifyS3CopyCompleted(podName, projectName);
128+
129+
// Deploy the model
130+
cy.step('Deploy the model');
131+
projectDetails.findSectionTab('model-server').click();
132+
modelServingGlobal.findSingleServingModelButton().click();
133+
modelServingGlobal.findDeployModelButton().click();
134+
inferenceServiceModal.findModelNameInput().type(modelName);
135+
inferenceServiceModal.findServingRuntimeTemplateSearchSelector().click();
136+
inferenceServiceModal.findGlobalScopedTemplateOption('OpenVINO Model Server').click();
137+
inferenceServiceModal.findModelFrameworkSelect().click();
138+
inferenceServiceModal.findOpenVinoIROpSet13().click();
139+
// There's only one PVC so it's automatically selected
140+
inferenceServiceModal.findExistingPVCConnectionOption().click();
141+
inferenceServiceModal.findSubmitButton().should('be.enabled').click();
142+
inferenceServiceModal.shouldBeOpen(false);
143+
144+
//Verify the model created and is running
145+
cy.step('Verify that the Model is running');
146+
checkInferenceServiceState(testData.singleModelName, projectName, {
147+
checkReady: true,
148+
checkLatestDeploymentReady: true,
149+
});
150+
},
151+
);
152+
});

‎frontend/src/__tests__/cypress/cypress/types.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,18 @@ export type PVCReplacements = {
8383
STORAGE_CLASS: string;
8484
};
8585

86+
export type PVCLoaderPodReplacements = {
87+
NAMESPACE: string;
88+
PVC_NAME: string;
89+
AWS_S3_BUCKET: string;
90+
AWS_DEFAULT_REGION: string;
91+
AWS_S3_ENDPOINT: string;
92+
AWS_ACCESS_KEY_ID: string;
93+
AWS_SECRET_ACCESS_KEY: string;
94+
POD_NAME: string;
95+
MODEL_PATH: string;
96+
};
97+
8698
export type WBEditTestData = {
8799
editTestNamespace: string;
88100
editedTestNamespace: string;

‎frontend/src/__tests__/cypress/cypress/utils/oc_commands/modelServing.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,3 +439,16 @@ export const validateInferenceServiceTolerations = (
439439
}
440440
});
441441
};
442+
443+
export const verifyS3CopyCompleted = (
444+
podName: string,
445+
namespace: string,
446+
): Cypress.Chainable<Cypress.Exec> => {
447+
return cy
448+
.exec(`oc logs ${podName} -n ${namespace}`, { failOnNonZeroExit: false })
449+
.then((result) => {
450+
if (!result.stdout.includes('S3 copy completed successfully')) {
451+
throw new Error('S3 copy did not complete successfully');
452+
}
453+
});
454+
};

‎frontend/src/__tests__/cypress/cypress/utils/oc_commands/pvcLoaderPod.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { replacePlaceholdersInYaml } from '#~/__tests__/cypress/cypress/utils/yaml_files';
2+
import type {
3+
CommandLineResult,
4+
PVCLoaderPodReplacements,
5+
} from '#~/__tests__/cypress/cypress/types';
6+
import { applyOpenShiftYaml } from './baseCommands';
7+
8+
export const createS3LoaderPod = (
9+
replacements: PVCLoaderPodReplacements,
10+
yamlFilePath = 'resources/yaml/pvc-loader-pod.yaml',
11+
): Cypress.Chainable<CommandLineResult> => {
12+
return cy.fixture(yamlFilePath).then((yamlContent) => {
13+
const modifiedYamlContent = replacePlaceholdersInYaml(yamlContent, replacements);
14+
cy.log('Creating S3 copy pod with YAML:', modifiedYamlContent);
15+
return applyOpenShiftYaml(modifiedYamlContent);
16+
});
17+
};

0 commit comments

Comments
 (0)