diff --git a/packages/cypress/cypress/fixtures/e2e/promptManagement/mlflowCR.yaml b/packages/cypress/cypress/fixtures/e2e/mlflow/mlflowCR.yaml similarity index 100% rename from packages/cypress/cypress/fixtures/e2e/promptManagement/mlflowCR.yaml rename to packages/cypress/cypress/fixtures/e2e/mlflow/mlflowCR.yaml diff --git a/packages/cypress/cypress/fixtures/e2e/mlflowExperiments/testMlflowExperiments.yaml b/packages/cypress/cypress/fixtures/e2e/mlflowExperiments/testMlflowExperiments.yaml new file mode 100644 index 0000000000..1e9130ed41 --- /dev/null +++ b/packages/cypress/cypress/fixtures/e2e/mlflowExperiments/testMlflowExperiments.yaml @@ -0,0 +1,22 @@ +projectName: 'mlflow-exp-test' +experiments: + - name: 'e2e-test-experiment' + renamedName: 'e2e-renamed-experiment' + - name: 'e2e-compare-experiment' + renamedName: 'e2e-compare-experiment-renamed' +runs: + - name: 'e2e-test-run-1' + parameters: + learning_rate: '0.01' + batch_size: '32' + metrics: + accuracy: '0.95' + loss: '0.05' + - name: 'e2e-test-run-2' + parameters: + learning_rate: '0.001' + batch_size: '64' + metrics: + accuracy: '0.97' + loss: '0.03' +nonExistentExperiment: 'nonexistent-experiment-xyz-no-results' diff --git a/packages/cypress/cypress/pages/appChrome.ts b/packages/cypress/cypress/pages/appChrome.ts index 3282c1794c..62f7cbcd85 100644 --- a/packages/cypress/cypress/pages/appChrome.ts +++ b/packages/cypress/cypress/pages/appChrome.ts @@ -43,6 +43,14 @@ class AppChrome { findNavItem(args: { name: string; rootSection?: string; subSection?: string }) { return this.findSideBar().findAppNavItem(args); } + + findDarkThemeToggle() { + return cy.findByTestId('dark-theme-toggle'); + } + + findLightThemeToggle() { + return cy.findByTestId('light-theme-toggle'); + } } export const appChrome = new AppChrome(); diff --git a/packages/cypress/cypress/pages/mlflowExperiments.ts b/packages/cypress/cypress/pages/mlflowExperiments.ts new file mode 100644 index 0000000000..d2c239f914 --- /dev/null +++ b/packages/cypress/cypress/pages/mlflowExperiments.ts @@ -0,0 +1,215 @@ +import { appChrome } from './appChrome'; + +export enum ExperimentTypeToggle { + GEN_AI = 'GenAI', + MODEL_TRAINING = 'Model training', +} + +const MLFLOW_DARK_MODE_KEY = '_mlflow_dark_mode_toggle_enabled'; +const EXPERIMENTS_PATH = '/develop-train/mlflow/experiments'; + +class MlflowExperiments { + visit(workspace?: string) { + const qs = workspace ? `?workspace=${workspace}` : ''; + cy.visitWithLogin(`${EXPERIMENTS_PATH}${qs}`); + this.wait(); + } + + navigate() { + this.findNavItem().click(); + this.wait(); + } + + private wait() { + cy.findByTestId('app-page-title', { timeout: 30000 }).should('exist'); + cy.testA11y(); + } + + waitForEmbeddedContent() { + this.findExperimentsSearchInput().should('be.visible'); + } + + shouldHaveExperimentsUrl() { + cy.url().should('include', EXPERIMENTS_PATH).should('include', 'workspace='); + } + + shouldHaveWorkspace(workspace: string) { + cy.url().should('include', `workspace=${workspace}`); + } + + findNavItem() { + return appChrome.findNavItem({ + name: 'Experiments (MLflow)', + rootSection: 'Develop & train', + }); + } + + findNavSection() { + return appChrome.findNavSection('Develop & train'); + } + + findPageTitle() { + return cy.findByTestId('app-page-title'); + } + + findLaunchMlflowButton() { + return cy.findByTestId('mlflow-embedded-jump-link', { timeout: 10000 }); + } + + findMlflowUnavailableState() { + return cy.findByTestId('mlflow-unavailable-empty-state'); + } + + findErrorEmptyState() { + return cy.findByTestId('empty-state-title', { timeout: 10000 }); + } + + findProjectSelector() { + return cy.findByTestId('project-selector-toggle', { timeout: 30000 }); + } + + findProjectInDropdown(name: string) { + return cy.findByRole('menuitem', { name }); + } + + findBreadcrumb() { + return cy.findByRole('navigation', { name: 'Breadcrumb' }); + } + + findBreadcrumbItem(label: string) { + return this.findBreadcrumb().contains(label); + } + + findExperimentTypeToggleItem(label: string) { + return cy.contains('[role="button"][aria-pressed]', label); + } + + shouldHaveExperimentTypeSelected(label: string) { + this.findExperimentTypeToggleItem(label).should('have.attr', 'aria-pressed', 'true'); + } + + findUsageTab() { + return cy.findByRole('tab', { name: 'Usage' }); + } + + findQualityTab() { + return cy.findByRole('tab', { name: 'Quality' }); + } + + findToolCallsTab() { + return cy.findByRole('tab', { name: 'Tool calls' }); + } + + shouldHaveUsageTabSelected() { + this.findUsageTab().should('have.attr', 'aria-selected', 'true'); + } + + findEvaluationRunsLink() { + return cy.findByRole('link', { name: 'Evaluation runs' }); + } + + findExperimentsSearchInput() { + return cy.findByTestId('search-experiment-input', { timeout: 30000 }); + } + + findCreateExperimentButton() { + return cy.findByTestId('create-experiment-table-empty-state-button', { timeout: 30000 }); + } + + findExperimentInTable(name: string) { + return cy.findByRole('link', { name }); + } + + findCreateExperimentModal() { + return cy.findByTestId('mlflow-input-modal'); + } + + findExperimentNameInput() { + return this.findCreateExperimentModal().find('input').first(); + } + + findCreateDialogSubmitButton() { + return this.findCreateExperimentModal().findByRole('button', { name: 'Create' }); + } + + findExperimentDetailHeading(name: string) { + return cy.findByRole('heading', { name, timeout: 10000 }); + } + + findOverflowMenuTrigger() { + return cy.findByTestId('overflow-menu-trigger'); + } + + findRenameAction() { + return cy.findByTestId('rename'); + } + + findDeleteAction() { + return cy.findByTestId('delete'); + } + + findRenameInput() { + return this.findCreateExperimentModal().find('input').first(); + } + + findRenameSubmitButton() { + return this.findCreateExperimentModal().findByRole('button', { name: 'Save' }); + } + + findDeleteConfirmModal() { + return cy.findByTestId('confirm-modal'); + } + + findDeleteConfirmButton() { + return this.findDeleteConfirmModal().findByRole('button', { name: 'Delete' }); + } + + shouldHaveRunsTable() { + cy.findByTestId('sort-header-Run Name', { timeout: 10000 }).should('exist'); + } + + findRunInTable(runName: string) { + return cy.contains('[role="row"]', runName); + } + + findRunCheckbox(runName: string) { + return this.findRunInTable(runName).find('[role="checkbox"], input[type="checkbox"]').first(); + } + + findCompareButton() { + return cy.findByRole('button', { name: /compare/i }); + } + + findCompareRunsHeading() { + return cy.contains('Comparing'); + } + + findCompareRunsVisualizations() { + return cy.contains('Visualizations'); + } + + findCompareRunDetails() { + return cy.contains('Run details'); + } + + shouldContainText(text: string) { + cy.contains(text).should('be.visible'); + } + + findRunParameters() { + return cy.contains('Parameters', { timeout: 10000 }); + } + + findRunMetrics() { + return cy.contains('Metrics', { timeout: 10000 }); + } + + getMlflowDarkModeStorageValue(): Cypress.Chainable { + return cy.window().then((win) => { + const value = win.localStorage.getItem(MLFLOW_DARK_MODE_KEY); + return cy.wrap(value); + }); + } +} + +export const mlflowExperiments = new MlflowExperiments(); diff --git a/packages/cypress/cypress/pages/promptManagement.ts b/packages/cypress/cypress/pages/promptManagement.ts index bff43b7cc6..eb80c66b5f 100644 --- a/packages/cypress/cypress/pages/promptManagement.ts +++ b/packages/cypress/cypress/pages/promptManagement.ts @@ -36,7 +36,19 @@ class PromptManagement { } findProjectSelector() { - return cy.findByTestId('project-selector-dropdown'); + return cy.findByTestId('project-selector-toggle', { timeout: 30000 }); + } + + findProjectInDropdown(name: string) { + return cy.findByRole('menuitem', { name }); + } + + shouldHaveWorkspace(workspace: string) { + cy.url().should('include', `workspace=${workspace}`); + } + + findErrorEmptyState() { + return cy.findByTestId('empty-state-title', { timeout: 10000 }); } findPromptsSearchInput() { @@ -93,24 +105,12 @@ class PromptManagement { return cy.findByRole('radio', { name: 'Preview' }); } - findDarkThemeToggle() { - return cy.findByTestId('dark-theme-toggle'); - } - - findLightThemeToggle() { - return cy.findByTestId('light-theme-toggle'); - } - getMlflowDarkModeStorageValue(): Cypress.Chainable { return cy.window().then((win) => { const value = win.localStorage.getItem(MLFLOW_DARK_MODE_KEY); return cy.wrap(value); }); } - - getHtmlDarkModeClass(): Cypress.Chainable { - return cy.document().then((doc) => doc.documentElement.classList.contains('pf-v6-theme-dark')); - } } export const promptManagement = new PromptManagement(); diff --git a/packages/cypress/cypress/tests/e2e/mlflowExperiments/testMlflowExperiments.cy.ts b/packages/cypress/cypress/tests/e2e/mlflowExperiments/testMlflowExperiments.cy.ts new file mode 100644 index 0000000000..b4fa7950cd --- /dev/null +++ b/packages/cypress/cypress/tests/e2e/mlflowExperiments/testMlflowExperiments.cy.ts @@ -0,0 +1,311 @@ +import { HTPASSWD_CLUSTER_ADMIN_USER } from '../../../utils/e2eUsers'; +import { + enableMlflowFeatures, + disableMlflowFeatures, + isMlflowOperatorManaged, + doesMlflowCRExist, + createMlflowExperimentViaAPI, + deleteMlflowExperimentViaAPI, + getMlflowExperimentIdByName, + logMlflowRunsViaAPI, +} from '../../../utils/oc_commands/mlflow'; +import { deleteOpenShiftProject, createOpenShiftProject } from '../../../utils/oc_commands/project'; +import { retryableBefore } from '../../../utils/retryableHooks'; +import { generateTestUUID } from '../../../utils/uuidGenerator'; +import { loadMlflowExperimentsFixture } from '../../../utils/dataLoader'; +import { mlflowExperiments, ExperimentTypeToggle } from '../../../pages/mlflowExperiments'; +import { appChrome } from '../../../pages/appChrome'; +import type { MlflowExperimentsTestData } from '../../../types'; + +describe('Verify MLflow Experiments page', () => { + let testData: MlflowExperimentsTestData; + let projectName: string; + let operatorWasManaged = true; + let crExisted = true; + let runsExperimentId: string | undefined; + let uiExperimentName: string | undefined; + let uiExperimentDeleted = false; + const uuid = generateTestUUID(); + + retryableBefore(() => { + loadMlflowExperimentsFixture('e2e/mlflowExperiments/testMlflowExperiments.yaml') + .then((fixtureData) => { + testData = fixtureData; + projectName = `${fixtureData.projectName}-${uuid}`; + return deleteOpenShiftProject(projectName, { wait: true, ignoreNotFound: true }); + }) + .then(() => createOpenShiftProject(projectName)) + .then(() => + isMlflowOperatorManaged().then((v) => { + operatorWasManaged = v; + }), + ) + .then(() => + doesMlflowCRExist().then((v) => { + crExisted = v; + cy.step( + `Pre-test state: operator=${operatorWasManaged ? 'Managed' : 'Removed'}, CR=${ + crExisted ? 'exists' : 'absent' + }`, + ); + }), + ) + .then(() => { + cy.step('Enable all features required for MLflow Experiments'); + return enableMlflowFeatures(); + }); + }); + + after(() => { + if (uiExperimentName && !uiExperimentDeleted) { + getMlflowExperimentIdByName(projectName, uiExperimentName).then((id) => { + if (id) { + deleteMlflowExperimentViaAPI(projectName, id); + } + }); + } + if (runsExperimentId) { + deleteMlflowExperimentViaAPI(projectName, runsExperimentId); + } + disableMlflowFeatures(operatorWasManaged, crExisted); + deleteOpenShiftProject(projectName, { wait: false, ignoreNotFound: true }); + }); + + it( + 'Verify MLflow Experiments page', + { + tags: ['@Sanity', '@SanitySet1', '@MLflow', '@MLflowExperiments', '@NonConcurrent'], + }, + () => { + const experiment = testData.experiments[0]; + const experimentName = `${experiment.name}-${uuid}`; + uiExperimentName = experimentName; + const renamedExperimentName = `${experiment.renamedName}-${uuid}`; + const runsExperimentName = `${testData.experiments[1].name}-${uuid}`; + const [run1, run2] = testData.runs; + + // ======================================================================= + // Experiment list and search + // ======================================================================= + + cy.step('Navigate to experiments page with workspace'); + mlflowExperiments.visit(projectName); + + cy.step('Wait for embedded MLflow UI to load'); + mlflowExperiments.waitForEmbeddedContent(); + + cy.step('Verify embedded MLflow UI loaded'); + mlflowExperiments.findMlflowUnavailableState().should('not.exist'); + + // ======================================================================= + // Create experiment, detail view, breadcrumbs + // ======================================================================= + + cy.step('Click "Create Experiment" button'); + mlflowExperiments.findCreateExperimentButton().should('be.visible').and('be.enabled'); + mlflowExperiments.findCreateExperimentButton().click(); + + cy.step('Fill in experiment name'); + mlflowExperiments.findExperimentNameInput().should('be.visible').type(experimentName); + + cy.step('Submit the create experiment form'); + mlflowExperiments.findCreateDialogSubmitButton().click(); + + cy.step('Verify experiment detail heading is shown'); + mlflowExperiments.findExperimentDetailHeading(experimentName).should('be.visible'); + + cy.step('Verify experiment type toggle is visible'); + mlflowExperiments + .findExperimentTypeToggleItem(ExperimentTypeToggle.GEN_AI) + .should('be.visible'); + mlflowExperiments + .findExperimentTypeToggleItem(ExperimentTypeToggle.MODEL_TRAINING) + .should('be.visible'); + + cy.step('Verify GenAI toggle is selected by default'); + mlflowExperiments.shouldHaveExperimentTypeSelected(ExperimentTypeToggle.GEN_AI); + + cy.step('Verify GenAI tabs are visible'); + mlflowExperiments.findUsageTab().should('be.visible'); + mlflowExperiments.findQualityTab().should('be.visible'); + mlflowExperiments.findToolCallsTab().should('be.visible'); + mlflowExperiments.shouldHaveUsageTabSelected(); + + cy.step('Verify Evaluation runs link is visible'); + mlflowExperiments.findEvaluationRunsLink().should('be.visible'); + + cy.step('Switch to Model training toggle'); + mlflowExperiments.findExperimentTypeToggleItem(ExperimentTypeToggle.MODEL_TRAINING).click(); + mlflowExperiments.shouldHaveExperimentTypeSelected(ExperimentTypeToggle.MODEL_TRAINING); + + cy.step('Switch back to GenAI toggle'); + mlflowExperiments.findExperimentTypeToggleItem(ExperimentTypeToggle.GEN_AI).click(); + mlflowExperiments.shouldHaveExperimentTypeSelected(ExperimentTypeToggle.GEN_AI); + + cy.step('Verify breadcrumbs appear'); + mlflowExperiments.findBreadcrumb().scrollIntoView().should('be.visible'); + mlflowExperiments.findBreadcrumbItem('Experiments').should('be.visible'); + mlflowExperiments.findBreadcrumbItem(experimentName).should('be.visible'); + + cy.step('Switch to Model training to verify runs view'); + mlflowExperiments.findExperimentTypeToggleItem(ExperimentTypeToggle.MODEL_TRAINING).click(); + mlflowExperiments.shouldHaveExperimentTypeSelected(ExperimentTypeToggle.MODEL_TRAINING); + + cy.step('Verify runs table or empty runs state is visible'); + mlflowExperiments.shouldHaveRunsTable(); + + cy.step('Switch back to GenAI'); + mlflowExperiments.findExperimentTypeToggleItem(ExperimentTypeToggle.GEN_AI).click(); + + cy.step('Click "Experiments" in breadcrumbs to navigate back'); + mlflowExperiments.findBreadcrumbItem('Experiments').click(); + + cy.step('Verify experiment list is restored'); + mlflowExperiments.findExperimentsSearchInput().should('be.visible'); + + cy.step('Navigate away to home page'); + cy.visitWithLogin('/', HTPASSWD_CLUSTER_ADMIN_USER); + appChrome.findMainContent().should('be.visible'); + + cy.step('Navigate back to experiments page'); + mlflowExperiments.visit(projectName); + + cy.step('Verify experiment persists after navigation'); + mlflowExperiments.findExperimentInTable(experimentName).should('be.visible'); + + cy.step('Search for the created experiment'); + mlflowExperiments.findExperimentsSearchInput().clear().type(`${experimentName}{enter}`); + mlflowExperiments.findExperimentInTable(experimentName).should('be.visible'); + + cy.step('Clear search and submit'); + mlflowExperiments.findExperimentsSearchInput().clear().type('{enter}'); + + cy.step('Search for a non-existent experiment'); + mlflowExperiments + .findExperimentsSearchInput() + .clear() + .type(`${testData.nonExistentExperiment}{enter}`); + mlflowExperiments.findExperimentInTable(experimentName).should('not.exist'); + + cy.step('Clear search to restore list'); + mlflowExperiments.findExperimentsSearchInput().clear().type('{enter}'); + mlflowExperiments.findExperimentInTable(experimentName).should('be.visible'); + + // ======================================================================= + // Rename experiment + // ======================================================================= + + cy.step('Click experiment to open detail page'); + mlflowExperiments.findExperimentInTable(experimentName).click(); + + cy.step('Open overflow menu on detail page'); + mlflowExperiments.findOverflowMenuTrigger().click(); + + cy.step('Click rename action'); + mlflowExperiments.findRenameAction().click(); + + cy.step('Clear and type new name'); + mlflowExperiments.findRenameInput().should('be.visible').clear().type(renamedExperimentName); + + cy.step('Submit rename'); + mlflowExperiments.findRenameSubmitButton().click(); + mlflowExperiments.findExperimentDetailHeading(renamedExperimentName).should('be.visible'); + uiExperimentName = renamedExperimentName; + + cy.step('Navigate back to list to verify renamed experiment'); + mlflowExperiments.findBreadcrumbItem('Experiments').click({ force: true }); + mlflowExperiments.findExperimentsSearchInput().should('be.visible'); + mlflowExperiments.findExperimentInTable(renamedExperimentName).should('be.visible'); + + // ======================================================================= + // Delete experiment + // ======================================================================= + + cy.step('Click renamed experiment to open detail page'); + mlflowExperiments.findExperimentInTable(renamedExperimentName).click(); + + cy.step('Open overflow menu for deletion'); + mlflowExperiments.findOverflowMenuTrigger().click(); + + cy.step('Click delete action'); + mlflowExperiments.findDeleteAction().click(); + + cy.step('Confirm deletion'); + mlflowExperiments.findDeleteConfirmButton().click(); + + cy.step('Verify redirected to experiment list'); + mlflowExperiments.findExperimentsSearchInput().should('be.visible'); + + cy.step('Verify experiment removed from list'); + mlflowExperiments.findExperimentInTable(renamedExperimentName).should('not.exist'); + uiExperimentDeleted = true; + + // ======================================================================= + // Experiment runs and comparison + // ======================================================================= + + cy.step('Create experiment and log runs via API'); + createMlflowExperimentViaAPI(projectName, runsExperimentName).then((experimentId) => { + runsExperimentId = experimentId; + logMlflowRunsViaAPI(projectName, experimentId, testData.runs).then(() => { + cy.step('Navigate to experiment detail page'); + mlflowExperiments.visit(projectName); + mlflowExperiments.findExperimentInTable(runsExperimentName).click(); + + cy.step('Switch to Model training to see runs'); + mlflowExperiments + .findExperimentTypeToggleItem(ExperimentTypeToggle.MODEL_TRAINING) + .click(); + mlflowExperiments.shouldHaveExperimentTypeSelected(ExperimentTypeToggle.MODEL_TRAINING); + + cy.step('Verify both runs appear in the runs table'); + mlflowExperiments.findRunInTable(run1.name).should('be.visible'); + mlflowExperiments.findRunInTable(run2.name).should('be.visible'); + + cy.step('Click on run 1 to view details'); + mlflowExperiments.findRunInTable(run1.name).find('a').first().click(); + + cy.step('Verify run detail page loaded'); + mlflowExperiments.findExperimentDetailHeading(run1.name).should('be.visible'); + + cy.step('Verify run parameters are displayed'); + mlflowExperiments.findRunParameters().scrollIntoView().should('be.visible'); + + cy.step('Verify run metrics are displayed'); + mlflowExperiments.findRunMetrics().scrollIntoView().should('be.visible'); + + cy.step('Navigate back to experiment via breadcrumbs'); + mlflowExperiments.findBreadcrumbItem(runsExperimentName).click(); + + cy.step('Switch to Model training to see runs table'); + mlflowExperiments + .findExperimentTypeToggleItem(ExperimentTypeToggle.MODEL_TRAINING) + .click(); + mlflowExperiments.findRunInTable(run1.name).should('be.visible'); + + cy.step('Select both runs via checkboxes'); + mlflowExperiments.findRunCheckbox(run1.name).click({ force: true }); + mlflowExperiments.findRunCheckbox(run2.name).click({ force: true }); + + cy.step('Click compare button'); + mlflowExperiments.findCompareButton().click(); + + cy.step('Verify compare runs page loaded without errors'); + mlflowExperiments.findCompareRunsHeading().should('be.visible'); + + cy.step('Verify both runs shown in comparison'); + mlflowExperiments.shouldContainText(run1.name); + mlflowExperiments.shouldContainText(run2.name); + + cy.step('Verify visualizations section is displayed'); + mlflowExperiments.findCompareRunsVisualizations().should('be.visible'); + mlflowExperiments.shouldContainText('accuracy'); + mlflowExperiments.shouldContainText('learning_rate'); + + cy.step('Verify run details section is displayed'); + mlflowExperiments.findCompareRunDetails().should('be.visible'); + }); + }); + }, + ); +}); diff --git a/packages/cypress/cypress/tests/e2e/promptManagement/testPromptManagement.cy.ts b/packages/cypress/cypress/tests/e2e/promptManagement/testPromptManagement.cy.ts index 366f07964e..6749ff2674 100644 --- a/packages/cypress/cypress/tests/e2e/promptManagement/testPromptManagement.cy.ts +++ b/packages/cypress/cypress/tests/e2e/promptManagement/testPromptManagement.cy.ts @@ -2,6 +2,9 @@ import { HTPASSWD_CLUSTER_ADMIN_USER } from '../../../utils/e2eUsers'; import { enablePromptManagementFeatures, disablePromptManagementFeatures, + isMlflowOperatorManaged, + doesMlflowCRExist, + isGenAiEnabled, } from '../../../utils/oc_commands/mlflow'; import { deleteOpenShiftProject, createOpenShiftProject } from '../../../utils/oc_commands/project'; import { retryableBefore } from '../../../utils/retryableHooks'; @@ -14,6 +17,9 @@ import type { PromptManagementTestData } from '../../../types'; describe('Verify Prompt Management page', () => { let testData: PromptManagementTestData; let projectName: string; + let operatorWasManaged = true; + let crExisted = true; + let genAiWasEnabled = true; const uuid = generateTestUUID(); retryableBefore(() => { @@ -24,6 +30,26 @@ describe('Verify Prompt Management page', () => { return deleteOpenShiftProject(projectName, { wait: true, ignoreNotFound: true }); }) .then(() => createOpenShiftProject(projectName)) + .then(() => + isMlflowOperatorManaged().then((v) => { + operatorWasManaged = v; + }), + ) + .then(() => + doesMlflowCRExist().then((v) => { + crExisted = v; + }), + ) + .then(() => + isGenAiEnabled().then((v) => { + genAiWasEnabled = v; + cy.step( + `Pre-test state: operator=${operatorWasManaged ? 'Managed' : 'Removed'}, CR=${ + crExisted ? 'exists' : 'absent' + }, GenAI=${genAiWasEnabled ? 'enabled' : 'disabled'}`, + ); + }), + ) .then(() => { cy.step('Enable all features required for Prompt Management'); return enablePromptManagementFeatures(); @@ -31,14 +57,14 @@ describe('Verify Prompt Management page', () => { }); after(() => { - disablePromptManagementFeatures(); + disablePromptManagementFeatures(operatorWasManaged, crExisted, genAiWasEnabled); deleteOpenShiftProject(projectName, { wait: false, ignoreNotFound: true }); }); it( 'Create a prompt and verify it appears in the prompts table', { - tags: ['@Sanity', '@SanitySet1', '@PromptManagement', '@MLflow'], + tags: ['@Sanity', '@SanitySet1', '@PromptManagement', '@MLflow', '@NonConcurrent'], }, () => { const prompt = testData.prompts[0]; @@ -119,20 +145,6 @@ describe('Verify Prompt Management page', () => { cy.step('Verify the prompt persists after navigation'); promptManagement.findPromptInTable(prompt.name).should('be.visible'); - - cy.step('Toggle dark mode on'); - promptManagement.findDarkThemeToggle().click(); - - cy.step('Verify dark theme is applied'); - promptManagement.getHtmlDarkModeClass().should('equal', true); - promptManagement.getMlflowDarkModeStorageValue().should('equal', 'true'); - - cy.step('Toggle light mode back on'); - promptManagement.findLightThemeToggle().click(); - - cy.step('Verify light theme is restored'); - promptManagement.getHtmlDarkModeClass().should('equal', false); - promptManagement.getMlflowDarkModeStorageValue().should('equal', 'false'); }, ); }); diff --git a/packages/cypress/cypress/tests/mocked/mlflow/mlflowExperiments.cy.ts b/packages/cypress/cypress/tests/mocked/mlflow/mlflowExperiments.cy.ts new file mode 100644 index 0000000000..3c0b7735e3 --- /dev/null +++ b/packages/cypress/cypress/tests/mocked/mlflow/mlflowExperiments.cy.ts @@ -0,0 +1,111 @@ +import { + mockDashboardConfig, + mockK8sResourceList, + mockProjectK8sResource, +} from '@odh-dashboard/internal/__mocks__'; +import { mockDscStatus } from '@odh-dashboard/internal/__mocks__/mockDscStatus'; +import { DataScienceStackComponent } from '@odh-dashboard/internal/concepts/areas/types'; +import { ProjectModel } from '../../../utils/models'; +import { asProductAdminUser } from '../../../utils/mockUsers'; +import { interceptMlflowStatus } from '../../../utils/mlflowUtils'; +import { mlflowExperiments } from '../../../pages/mlflowExperiments'; +import { appChrome } from '../../../pages/appChrome'; + +const PROJECT_A = 'test-project-a'; +const PROJECT_B = 'test-project-b'; + +const initIntercepts = ({ mlflowConfigured = true }: { mlflowConfigured?: boolean } = {}) => { + asProductAdminUser(); + cy.interceptOdh('GET /api/config', mockDashboardConfig({})); + interceptMlflowStatus(mlflowConfigured); + + const projectA = mockProjectK8sResource({ k8sName: PROJECT_A, displayName: PROJECT_A }); + const projectB = mockProjectK8sResource({ k8sName: PROJECT_B, displayName: PROJECT_B }); + cy.interceptK8sList(ProjectModel, mockK8sResourceList([projectA, projectB])); + cy.interceptK8s(ProjectModel, projectA); +}; + +describe('MLflow Experiments page wrapper', () => { + beforeEach(() => { + initIntercepts(); + }); + + describe('Page chrome and navigation', () => { + it('should display page title and Launch MLflow button', () => { + mlflowExperiments.visit(PROJECT_A); + mlflowExperiments.findPageTitle().should('be.visible'); + mlflowExperiments + .findLaunchMlflowButton() + .should('be.visible') + .should('have.attr', 'href', '/mlflow') + .should('have.attr', 'target', '_blank'); + }); + + it('should navigate via sidebar and show active nav item', () => { + cy.visitWithLogin('/'); + appChrome.findMainContent().should('be.visible'); + + mlflowExperiments.findNavSection().click(); + mlflowExperiments.navigate(); + + mlflowExperiments.shouldHaveExperimentsUrl(); + mlflowExperiments.findNavItem().should('have.attr', 'aria-current', 'page'); + }); + }); + + describe('Project selector', () => { + it('should switch workspace when selecting a different project', () => { + mlflowExperiments.visit(PROJECT_A); + mlflowExperiments.findProjectSelector().should('contain', PROJECT_A); + + mlflowExperiments.findProjectSelector().click(); + mlflowExperiments.findProjectInDropdown(PROJECT_B).click(); + mlflowExperiments.shouldHaveWorkspace(PROJECT_B); + + mlflowExperiments.findProjectSelector().click(); + mlflowExperiments.findProjectInDropdown(PROJECT_A).click(); + mlflowExperiments.shouldHaveWorkspace(PROJECT_A); + }); + }); + + describe('Dark mode toggle', () => { + it('should sync localStorage on toggle', () => { + mlflowExperiments.visit(PROJECT_A); + + appChrome.findDarkThemeToggle().click(); + mlflowExperiments.getMlflowDarkModeStorageValue().should('equal', 'true'); + + appChrome.findLightThemeToggle().click(); + mlflowExperiments.getMlflowDarkModeStorageValue().should('equal', 'false'); + }); + }); + + describe('Error states', () => { + it('should show error state for invalid workspace', () => { + const invalidWorkspace = 'nonexistent-project'; + mlflowExperiments.visit(invalidWorkspace); + mlflowExperiments + .findErrorEmptyState() + .should('be.visible') + .should('contain', invalidWorkspace); + }); + + it('should show unavailable state when MLflow is not configured', () => { + initIntercepts({ mlflowConfigured: false }); + mlflowExperiments.visit(PROJECT_A); + mlflowExperiments.findMlflowUnavailableState().should('be.visible'); + }); + + it('should hide nav item when MLflow operator is removed', () => { + const dscStatus = mockDscStatus({}); + dscStatus.components = { + ...dscStatus.components, + [DataScienceStackComponent.MLFLOW]: { managementState: 'Removed' }, + }; + cy.interceptOdh('GET /api/dsc/status', dscStatus); + + cy.visitWithLogin('/'); + mlflowExperiments.findNavItem().should('not.exist'); + }); + }); +}); diff --git a/packages/cypress/cypress/tests/mocked/mlflow/promptManagement.cy.ts b/packages/cypress/cypress/tests/mocked/mlflow/promptManagement.cy.ts new file mode 100644 index 0000000000..cf1890175d --- /dev/null +++ b/packages/cypress/cypress/tests/mocked/mlflow/promptManagement.cy.ts @@ -0,0 +1,109 @@ +import { + mockDashboardConfig, + mockK8sResourceList, + mockProjectK8sResource, +} from '@odh-dashboard/internal/__mocks__'; +import { mockDscStatus } from '@odh-dashboard/internal/__mocks__/mockDscStatus'; +import { DataScienceStackComponent } from '@odh-dashboard/internal/concepts/areas/types'; +import { ProjectModel } from '../../../utils/models'; +import { asProductAdminUser } from '../../../utils/mockUsers'; +import { interceptMlflowStatus } from '../../../utils/mlflowUtils'; +import { promptManagement } from '../../../pages/promptManagement'; +import { appChrome } from '../../../pages/appChrome'; + +const PROJECT_A = 'test-project-a'; +const PROJECT_B = 'test-project-b'; + +const initIntercepts = ({ + mlflowConfigured = true, + genAiStudio = true, +}: { mlflowConfigured?: boolean; genAiStudio?: boolean } = {}) => { + asProductAdminUser(); + cy.interceptOdh('GET /api/config', mockDashboardConfig({ genAiStudio })); + interceptMlflowStatus(mlflowConfigured); + + const projectA = mockProjectK8sResource({ k8sName: PROJECT_A, displayName: PROJECT_A }); + const projectB = mockProjectK8sResource({ k8sName: PROJECT_B, displayName: PROJECT_B }); + cy.interceptK8sList(ProjectModel, mockK8sResourceList([projectA, projectB])); + cy.interceptK8s(ProjectModel, projectA); +}; + +describe('Prompt Management page wrapper', () => { + beforeEach(() => { + initIntercepts(); + }); + + describe('Page chrome', () => { + it('should display page title and Launch MLflow button', () => { + promptManagement.visit(PROJECT_A); + promptManagement.findPageTitle().should('be.visible'); + promptManagement + .findLaunchMlflowButton() + .should('be.visible') + .should('have.attr', 'href', '/mlflow') + .should('have.attr', 'target', '_blank'); + }); + }); + + describe('Project selector', () => { + it('should switch workspace when selecting a different project', () => { + promptManagement.visit(PROJECT_A); + promptManagement.findProjectSelector().should('contain', PROJECT_A); + + promptManagement.findProjectSelector().click(); + promptManagement.findProjectInDropdown(PROJECT_B).click(); + promptManagement.shouldHaveWorkspace(PROJECT_B); + + promptManagement.findProjectSelector().click(); + promptManagement.findProjectInDropdown(PROJECT_A).click(); + promptManagement.shouldHaveWorkspace(PROJECT_A); + }); + }); + + describe('Dark mode toggle', () => { + it('should sync localStorage on toggle', () => { + promptManagement.visit(PROJECT_A); + + appChrome.findDarkThemeToggle().click(); + promptManagement.getMlflowDarkModeStorageValue().should('equal', 'true'); + + appChrome.findLightThemeToggle().click(); + promptManagement.getMlflowDarkModeStorageValue().should('equal', 'false'); + }); + }); + + describe('Error states', () => { + it('should show error state for invalid workspace', () => { + const invalidWorkspace = 'nonexistent-project'; + promptManagement.visit(invalidWorkspace); + promptManagement + .findErrorEmptyState() + .should('be.visible') + .should('contain', invalidWorkspace); + }); + + it('should show unavailable state when MLflow is not configured', () => { + initIntercepts({ mlflowConfigured: false }); + promptManagement.visit(PROJECT_A); + promptManagement.findMlflowUnavailableState().should('be.visible'); + }); + + it('should hide nav item when genAiStudio feature flag is disabled', () => { + initIntercepts({ genAiStudio: false }); + cy.visitWithLogin('/'); + promptManagement.findNavItem().should('not.exist'); + }); + + it('should hide nav item when MLflow operator is removed', () => { + const dscStatus = mockDscStatus({}); + dscStatus.components = { + ...dscStatus.components, + [DataScienceStackComponent.MLFLOW]: { managementState: 'Removed' }, + }; + cy.interceptOdh('GET /api/dsc/status', dscStatus); + + cy.visitWithLogin('/'); + promptManagement.findNavItem().should('not.exist'); + }); + }); +}); diff --git a/packages/cypress/cypress/types.ts b/packages/cypress/cypress/types.ts index 00f7de91ba..8177f42660 100644 --- a/packages/cypress/cypress/types.ts +++ b/packages/cypress/cypress/types.ts @@ -700,3 +700,21 @@ export type PromptManagementTestData = { projectName: string; prompts: PromptManagementPromptData[]; }; + +export type MlflowExperimentRunData = { + name: string; + parameters: Record; + metrics: Record; +}; + +export type MlflowExperimentData = { + name: string; + renamedName: string; +}; + +export type MlflowExperimentsTestData = { + projectName: string; + experiments: MlflowExperimentData[]; + runs: MlflowExperimentRunData[]; + nonExistentExperiment: string; +}; diff --git a/packages/cypress/cypress/utils/dataLoader.ts b/packages/cypress/cypress/utils/dataLoader.ts index f79144bbe1..7b091b122a 100644 --- a/packages/cypress/cypress/utils/dataLoader.ts +++ b/packages/cypress/cypress/utils/dataLoader.ts @@ -23,6 +23,7 @@ import type { WorkloadMetricsTestData, KueueWorkbenchTestData, PromptManagementTestData, + MlflowExperimentsTestData, } from '../types'; // Load fixture function that returns DataScienceProjectData @@ -212,3 +213,12 @@ export const loadPromptManagementFixture = ( return data; }); + +export const loadMlflowExperimentsFixture = ( + fixturePath: string, +): Cypress.Chainable => + cy.fixture(fixturePath, 'utf8').then((yamlContent: string) => { + const data = yaml.load(yamlContent) as MlflowExperimentsTestData; + + return data; + }); diff --git a/packages/cypress/cypress/utils/mlflowUtils.ts b/packages/cypress/cypress/utils/mlflowUtils.ts new file mode 100644 index 0000000000..c1e416fb35 --- /dev/null +++ b/packages/cypress/cypress/utils/mlflowUtils.ts @@ -0,0 +1,5 @@ +export const MLFLOW_BFF_STATUS_URL = '/_bff/mlflow/api/v1/status'; + +export const interceptMlflowStatus = (configured = true): void => { + cy.intercept('GET', MLFLOW_BFF_STATUS_URL, { body: { configured } }).as('mlflowStatus'); +}; diff --git a/packages/cypress/cypress/utils/oc_commands/genAi.ts b/packages/cypress/cypress/utils/oc_commands/genAi.ts index 0800768c05..d1a13a5bbf 100644 --- a/packages/cypress/cypress/utils/oc_commands/genAi.ts +++ b/packages/cypress/cypress/utils/oc_commands/genAi.ts @@ -10,9 +10,8 @@ const DASHBOARD_CONFIG = 'odhdashboardconfig odh-dashboard-config'; // Polling configuration for UI visibility (slower, requires page reload) const UI_POLL_CONFIG = { - maxAttempts: 15, - pollIntervalMs: 10000, // 10 seconds between attempts - pageLoadWaitMs: 5000, + maxAttempts: 20, + pollIntervalMs: 5000, } as const; /** @@ -87,69 +86,83 @@ const waitForGenAiStudioFeatureFlag = (): Cypress.Chainable => { * * @returns A Cypress chainable resolving to true if the section exists, false otherwise. */ -const isGenAiStudioVisible = (): Cypress.Chainable => { - return appChrome - .findSideBar() - .then(($sidebar) => $sidebar.find('button:contains("Gen AI studio")').length > 0); -}; - /** - * Poll until the Gen AI Studio section appears in the sidebar. - * Refreshes the page and checks for the nav section on each attempt. - * - * @returns A Cypress chainable that resolves when the section is visible. + * Retry-aware check for an element inside the sidebar. + * See `findNavItemInSidebar` in mlflow.ts for rationale. */ +const SIDEBAR_SETTLE_TIMEOUT = 30000; +const NAV_ITEM_SETTLE_MS = 15000; +const NAV_ITEM_POLL_MS = 500; + +const findInSidebar = (selector: string): Cypress.Chainable => + appChrome.findSideBar().then( + ($sidebar) => + new Cypress.Promise((resolve) => { + const deadline = Date.now() + NAV_ITEM_SETTLE_MS; + const poll = () => { + if ($sidebar.find(selector).length > 0) { + resolve(true); + } else if (Date.now() >= deadline) { + resolve(false); + } else { + setTimeout(poll, NAV_ITEM_POLL_MS); + } + }; + poll(); + }), + ); + const waitForGenAiStudioInSidebar = (): Cypress.Chainable => { - const { maxAttempts, pollIntervalMs, pageLoadWaitMs } = UI_POLL_CONFIG; + const { maxAttempts, pollIntervalMs } = UI_POLL_CONFIG; const startTime = Date.now(); const checkForSection = (attemptNumber = 1): Cypress.Chainable => { cy.step(`Attempt ${attemptNumber}/${maxAttempts} - Checking for Gen AI studio in sidebar...`); - // Visit/reload the dashboard to get fresh sidebar state - cy.visitWithLogin('/'); + if (attemptNumber === 1) { + cy.visitWithLogin('/'); + } else { + cy.reload(); + } - // Wait for page to fully load, then check for the section - // eslint-disable-next-line cypress/no-unnecessary-waiting - return cy.wait(pageLoadWaitMs).then(() => { - return isGenAiStudioVisible().then((isVisible) => { - const elapsedTime = ((Date.now() - startTime) / 1000).toFixed(1); + cy.get('[data-testid="dashboard-page-main"]', { timeout: SIDEBAR_SETTLE_TIMEOUT }); - if (isVisible) { - cy.log(`✅ Gen AI studio section found in sidebar (after ${elapsedTime}s)`); - return cy.wrap(true); - } + return findInSidebar('button:contains("Gen AI studio")').then((isVisible) => { + const elapsedTime = ((Date.now() - startTime) / 1000).toFixed(1); - if (attemptNumber >= maxAttempts) { - throw new Error( - `Gen AI studio section not found in sidebar after ${maxAttempts} attempts (${elapsedTime}s)`, - ); - } + if (isVisible) { + cy.log(`Gen AI studio section found in sidebar (after ${elapsedTime}s)`); + return cy.wrap(true); + } - cy.log( - `⏳ Gen AI studio not yet visible (attempt ${attemptNumber}/${maxAttempts}, elapsed: ${elapsedTime}s)`, + if (attemptNumber >= maxAttempts) { + throw new Error( + `Gen AI studio section not found in sidebar after ${maxAttempts} attempts (${elapsedTime}s)`, ); + } - // eslint-disable-next-line cypress/no-unnecessary-waiting - return cy.wait(pollIntervalMs).then(() => checkForSection(attemptNumber + 1)); - }); + cy.log( + `Gen AI studio not yet visible (attempt ${attemptNumber}/${maxAttempts}, elapsed: ${elapsedTime}s)`, + ); + + // eslint-disable-next-line cypress/no-unnecessary-waiting + return cy.wait(pollIntervalMs).then(() => checkForSection(attemptNumber + 1)); }); }; const totalTimeout = (maxAttempts * pollIntervalMs) / 1000; - cy.log(`🔍 Polling for Gen AI studio in sidebar (max ${totalTimeout}s)`); + cy.log(`Polling for Gen AI studio in sidebar (max ${totalTimeout}s)`); return checkForSection(); }; /** - * Enable Gen AI features by patching the DataScienceCluster and ODHDashboardConfig resources. - * Sets LlamaStack operator to Managed, waits for the operator to be ready, - * waits for the namespace, enables Gen AI Studio, polls for the feature flag, - * and finally polls until it appears in the sidebar. + * Enable Gen AI backend resources without waiting for sidebar visibility. + * Sets LlamaStack operator to Managed, waits for readiness, namespace, + * enables Gen AI Studio flag, and polls for the feature flag. * - * @returns A Cypress chainable that resolves when Gen AI Studio is visible in the sidebar. + * Useful for composition when a caller will perform its own sidebar check. */ -export const enableGenAiFeatures = (): Cypress.Chainable => { +export const enableGenAiBackend = (): Cypress.Chainable => { const namespace = getApplicationsNamespace(); cy.step('Set LlamaStack to Managed'); @@ -169,6 +182,33 @@ export const enableGenAiFeatures = (): Cypress.Chainable => { .then(() => { cy.step('Wait for genAiStudio feature flag to be set'); return waitForGenAiStudioFeatureFlag(); + }); +}; + +/** + * Poll until the DSC status reports llamastackoperator as Managed. + * The dashboard frontend reads DSC status to evaluate the plugin-gen-ai area flag. + */ +const waitForDSCLlamaStackManaged = (): Cypress.Chainable => + pollUntilSuccess( + `oc get ${DSC_RESOURCE} -o json | jq -e '.status.components.llamastackoperator.managementState == "Managed"'`, + 'DSC status to reflect llamastackoperator as Managed', + { maxAttempts: 60, pollIntervalMs: 5000 }, + ); + +/** + * Enable Gen AI features by patching the DataScienceCluster and ODHDashboardConfig resources. + * Sets LlamaStack operator to Managed, waits for the operator to be ready, + * waits for the namespace, enables Gen AI Studio, polls for the feature flag, + * verifies DSC status, and finally polls until it appears in the sidebar. + * + * @returns A Cypress chainable that resolves when Gen AI Studio is visible in the sidebar. + */ +export const enableGenAiFeatures = (): Cypress.Chainable => { + return enableGenAiBackend() + .then(() => { + cy.step('Verify DSC status reflects llamastackoperator as Managed'); + return waitForDSCLlamaStackManaged(); }) .then(() => { cy.step('Wait for Gen AI Studio to appear in sidebar'); diff --git a/packages/cypress/cypress/utils/oc_commands/mlflow.ts b/packages/cypress/cypress/utils/oc_commands/mlflow.ts index 5bf64aaab1..206133381b 100644 --- a/packages/cypress/cypress/utils/oc_commands/mlflow.ts +++ b/packages/cypress/cypress/utils/oc_commands/mlflow.ts @@ -1,7 +1,7 @@ import { pollUntilSuccess } from './baseCommands'; -import { enableGenAiFeatures, disableGenAiFeatures } from './genAi'; +import { enableGenAiBackend, disableGenAiFeatures } from './genAi'; import { appChrome } from '../../pages/appChrome'; -import type { CommandLineResult } from '../../types'; +import type { CommandLineResult, MlflowExperimentRunData } from '../../types'; import { maskSensitiveInfo } from '../maskSensitiveInfo'; const DSC_RESOURCE = 'datasciencecluster default-dsc'; @@ -9,8 +9,7 @@ const K8S_NAMESPACE_RE = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/; const UI_POLL_CONFIG = { maxAttempts: 20, - pollIntervalMs: 10000, - pageLoadWaitMs: 5000, + pollIntervalMs: 5000, } as const; const assertNamespace = (namespace: string): string => { @@ -28,6 +27,56 @@ const getApplicationsNamespace = (): string => { return assertNamespace(namespace); }; +/** + * Check if the MLflow operator is currently set to Managed in the DSC. + */ +export const isMlflowOperatorManaged = (): Cypress.Chainable => + cy + .exec( + `oc get ${DSC_RESOURCE} -o jsonpath="{.spec.components.mlflowoperator.managementState}"`, + { failOnNonZeroExit: false }, + ) + .then((result) => { + if (result.code !== 0) { + throw new Error(`Failed to read mlflowoperator state: ${maskSensitiveInfo(result.stderr)}`); + } + return result.stdout.replace(/"/g, '').trim() === 'Managed'; + }); + +/** + * Check if Gen AI (LlamaStack operator) is currently set to Managed in the DSC. + */ +export const isGenAiEnabled = (): Cypress.Chainable => + cy + .exec( + `oc get ${DSC_RESOURCE} -o jsonpath="{.spec.components.llamastackoperator.managementState}"`, + { failOnNonZeroExit: false }, + ) + .then((result) => { + if (result.code !== 0) { + throw new Error( + `Failed to read llamastackoperator state: ${maskSensitiveInfo(result.stderr)}`, + ); + } + return result.stdout.replace(/"/g, '').trim() === 'Managed'; + }); + +/** + * Check if an MLflow CR exists in the applications namespace. + */ +export const doesMlflowCRExist = (): Cypress.Chainable => + cy + .exec( + `oc get mlflows.mlflow.opendatahub.io -n ${getApplicationsNamespace()} --no-headers 2>/dev/null | head -1`, + { failOnNonZeroExit: false }, + ) + .then((result) => { + if (result.code !== 0 && !result.stderr.includes('No resources found')) { + throw new Error(`Failed to check MLflow CR: ${maskSensitiveInfo(result.stderr)}`); + } + return result.stdout.trim().length > 0; + }); + const buildPatchCommand = (resource: string, patchJson: object, namespace?: string): string => { const safeNamespace = namespace ? assertNamespace(namespace) : undefined; const namespaceFlag = safeNamespace ? ` -n ${safeNamespace}` : ''; @@ -82,7 +131,7 @@ const ensureMlflowCR = (namespace: string): Cypress.Chainable cy.log('Creating MLflow CR...'); return cy - .exec(`oc apply -n ${safeNamespace} -f cypress/fixtures/e2e/promptManagement/mlflowCR.yaml`, { + .exec(`oc apply -n ${safeNamespace} -f cypress/fixtures/e2e/mlflow/mlflowCR.yaml`, { failOnNonZeroExit: false, }) .then((applyResult) => { @@ -132,70 +181,288 @@ const waitForMlflowCRReady = (namespace: string): Cypress.Chainable => - appChrome.findSideBar().then(($sidebar) => $sidebar.find('a:contains("Prompts")').length > 0); +const SIDEBAR_SETTLE_TIMEOUT = 30000; + +const logSidebarDiagnostics = (): void => { + cy.window({ log: false }).then((win) => { + const mfEl = win.document.getElementById('mf-remotes-json'); + const mfContent = mfEl?.textContent ?? '(element not found)'; + cy.log(`[DIAG] mf-remotes-json: ${mfContent}`); + }); + + appChrome.findSideBar().then(($sidebar) => { + const links: string[] = []; + $sidebar.find('a').each((_i, el) => { + const text = el.textContent; + if (text && text.trim()) { + links.push(text.trim()); + } + }); + cy.log(`[DIAG] Sidebar links (${links.length}): ${links.join(' | ')}`); + + const sections: string[] = []; + $sidebar.find('button').each((_i, el) => { + const text = el.textContent; + if (text && text.trim()) { + sections.push(text.trim()); + } + }); + cy.log(`[DIAG] Sidebar sections (${sections.length}): ${sections.join(' | ')}`); + }); + + cy.request({ url: '/api/dsc/status', failOnStatusCode: false, timeout: 10000, log: false }).then( + (resp) => { + if (resp.status === 200 && resp.body?.components) { + const mlflow = resp.body.components.mlflowoperator?.managementState ?? '(missing)'; + const llama = resp.body.components.llamastackoperator?.managementState ?? '(missing)'; + cy.log(`[DIAG] /api/dsc/status: mlflowoperator=${mlflow}, llamastackoperator=${llama}`); + } else { + cy.log(`[DIAG] /api/dsc/status: HTTP ${resp.status}`); + } + }, + ); +}; /** - * Poll until the Prompts nav item appears in the sidebar. + * Retry-aware check for a nav item inside the sidebar element. + * + * After `dashboard-page-main` appears, area flags are set via a React + * useEffect that runs asynchronously. Extension-provided nav items only + * render once the PluginStore has evaluated those flags. This creates a + * short gap (typically < 1 s) during which the sidebar exists but doesn't + * yet contain extension items. A one-shot jQuery `.find()` during that gap + * always misses the item, making the outer reload loop retry needlessly. + * + * This helper polls the live jQuery element for up to `NAV_ITEM_SETTLE_MS` + * so that a single page load is enough once the extensions are ready. */ -const waitForPromptsInSidebar = (): Cypress.Chainable => { - const { maxAttempts, pollIntervalMs, pageLoadWaitMs } = UI_POLL_CONFIG; +const NAV_ITEM_SETTLE_MS = 15000; +const NAV_ITEM_POLL_MS = 500; + +const findNavItemInSidebar = ($sidebar: JQuery, navLabel: string): Cypress.Chainable => + cy.wrap(null, { log: false }).then( + () => + new Cypress.Promise((resolve) => { + const deadline = Date.now() + NAV_ITEM_SETTLE_MS; + const poll = () => { + if ($sidebar.find(`a:contains("${navLabel}")`).length > 0) { + resolve(true); + } else if (Date.now() >= deadline) { + resolve(false); + } else { + setTimeout(poll, NAV_ITEM_POLL_MS); + } + }; + poll(); + }), + ); + +const waitForNavItemInSidebar = (navLabel: string): Cypress.Chainable => { + const { maxAttempts, pollIntervalMs } = UI_POLL_CONFIG; const startTime = Date.now(); const check = (attemptNumber = 1): Cypress.Chainable => { - cy.step(`Attempt ${attemptNumber}/${maxAttempts} - Checking for Prompts in sidebar...`); + cy.step(`Attempt ${attemptNumber}/${maxAttempts} - Checking for ${navLabel} in sidebar...`); - cy.visitWithLogin('/'); + if (attemptNumber === 1) { + cy.visitWithLogin('/'); + } else { + cy.reload(); + } - // eslint-disable-next-line cypress/no-unnecessary-waiting - return cy.wait(pageLoadWaitMs).then(() => - isPromptsNavVisible().then((isVisible) => { + cy.get('[data-testid="dashboard-page-main"]', { timeout: SIDEBAR_SETTLE_TIMEOUT }); + + return appChrome + .findSideBar() + .then(($sidebar) => findNavItemInSidebar($sidebar, navLabel)) + .then((found) => { const elapsedTime = ((Date.now() - startTime) / 1000).toFixed(1); - if (isVisible) { - cy.log(`Prompts nav item found in sidebar (after ${elapsedTime}s)`); + if (found) { + cy.log(`${navLabel} nav item found in sidebar (after ${elapsedTime}s)`); return cy.wrap(true); } + if (attemptNumber === 1) { + logSidebarDiagnostics(); + } + if (attemptNumber >= maxAttempts) { + logSidebarDiagnostics(); throw new Error( - `Prompts nav item not found in sidebar after ${maxAttempts} attempts (${elapsedTime}s)`, + `${navLabel} nav item not found in sidebar after ${maxAttempts} attempts (${elapsedTime}s)`, ); } cy.log( - `Prompts not yet visible (attempt ${attemptNumber}/${maxAttempts}, elapsed: ${elapsedTime}s)`, + `${navLabel} not yet visible (attempt ${attemptNumber}/${maxAttempts}, elapsed: ${elapsedTime}s)`, ); // eslint-disable-next-line cypress/no-unnecessary-waiting return cy.wait(pollIntervalMs).then(() => check(attemptNumber + 1)); - }), - ); + }); }; const totalTimeout = (maxAttempts * pollIntervalMs) / 1000; - cy.log(`Polling for Prompts in sidebar (max ${totalTimeout}s)`); + cy.log(`Polling for ${navLabel} in sidebar (max ${totalTimeout}s)`); return check(); }; /** - * Enable all features required for Prompt Management: - * 1. Enable Gen AI features (LlamaStack operator, genAiStudio flag, sidebar) - * 2. Set mlflowoperator to Managed and wait for it - * 3. Create an MLflow CR and wait for it to be ready - * 4. Wait for Prompts nav item in the sidebar + * Poll until the DSC status reports given components as Managed. + * The dashboard frontend reads DSC status to evaluate area flags, so these + * must be reflected in the cluster status before the nav items will appear. */ -export const enablePromptManagementFeatures = (): Cypress.Chainable => { +const waitForDSCComponentsManaged = (components: string[]): Cypress.Chainable => { + const jqChecks = components + .map((c) => `.components.${c}.managementState == "Managed"`) + .join(' and '); + const command = `oc get datasciencecluster default-dsc -o json | jq -e '.status | ${jqChecks}'`; + return pollUntilSuccess(command, `DSC components [${components.join(', ')}] to be Managed`, { + maxAttempts: 60, + pollIntervalMs: 5000, + }); +}; + +/** + * Poll until the MLflow tracking server pod is Ready (not just Running). + * The CR status.address.url can be set before the pod passes readiness probes, + * which means the BFF may still return 503 until the pod is fully ready. + */ +const waitForMlflowTrackingPodReady = (namespace: string): Cypress.Chainable => { + const ns = assertNamespace(namespace); + return pollUntilSuccess( + `oc wait --for=condition=Available deployment/mlflow -n ${ns} --timeout=0`, + 'MLflow tracking server deployment to be Available', + { maxAttempts: 60, pollIntervalMs: 5000 }, + ); +}; + +/** + * Poll the dashboard backend's /api/dsc/status until it reflects the expected + * component management states. + * + * The backend uses a ResourceWatcher that caches DSC status with a 2-minute + * (active) or 30-minute (inactive) polling interval. After patching the DSC + * on the cluster, the cached response can be stale. Polling this endpoint: + * - activates the watcher (switches from 30 min to 2 min interval) + * - waits for the cache to refresh before the frontend evaluates area flags + * + * Requires an active browser session (call after cy.visitWithLogin). + */ +const waitForDashboardDSCStatus = ( + components: Record, +): Cypress.Chainable => { + const maxAttempts = 60; + const pollIntervalMs = 5000; + const startTime = Date.now(); + const label = Object.entries(components) + .map(([k, v]) => `${k}=${v}`) + .join(', '); + + const check = (attempt = 1): Cypress.Chainable => { + cy.step(`Attempt ${attempt}/${maxAttempts} - Polling /api/dsc/status for ${label}...`); + + return cy + .request({ url: '/api/dsc/status', failOnStatusCode: false, timeout: 10000 }) + .then((resp) => { + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + + if (resp.status === 200 && resp.body?.components) { + const allMatch = Object.entries(components).every( + ([comp, expected]) => resp.body.components[comp]?.managementState === expected, + ); + if (allMatch) { + cy.log(`Dashboard DSC status matches (after ${elapsed}s)`); + return cy.wrap(true); + } + } + + if (attempt >= maxAttempts) { + throw new Error( + `Dashboard /api/dsc/status did not reflect ${label} after ${maxAttempts} ` + + `attempts (${elapsed}s)`, + ); + } + + cy.log( + `Dashboard DSC status not yet updated (attempt ${attempt}/${maxAttempts}, ${elapsed}s)`, + ); + // eslint-disable-next-line cypress/no-unnecessary-waiting + return cy.wait(pollIntervalMs).then(() => check(attempt + 1)); + }); + }; + + const totalTimeout = (maxAttempts * pollIntervalMs) / 1000; + cy.log(`Polling dashboard /api/dsc/status for ${label} (max ${totalTimeout}s)`); + return check(); +}; + +/** + * Poll via cy.request() until the mlflowEmbedded module federation remote entry + * returns HTTP 200. The dashboard proxies this through /_mf/mlflowEmbedded/*. + * + * The k8s deployment can report "Available" before the MLflow web server is + * ready to serve the federated JavaScript bundle. Without this check the + * sidebar poll would start while loadRemote() still fails silently, so the + * nav item never appears. + * + * Requires an active browser session (call after cy.visitWithLogin). + */ +const waitForMlflowRemoteEntry = (): Cypress.Chainable => { + const remoteEntryPath = '/_mf/mlflowEmbedded/mlflow/static-files/federated/remoteEntry.js'; + const maxAttempts = 60; + const pollIntervalMs = 5000; + const startTime = Date.now(); + + const check = (attempt = 1): Cypress.Chainable => { + cy.step(`Attempt ${attempt}/${maxAttempts} - Polling mlflowEmbedded remote entry...`); + + return cy + .request({ url: remoteEntryPath, failOnStatusCode: false, timeout: 10000 }) + .then((resp) => { + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + + if (resp.status === 200) { + cy.log(`mlflowEmbedded remote entry reachable (after ${elapsed}s)`); + return cy.wrap(true); + } + + if (attempt >= maxAttempts) { + throw new Error( + `mlflowEmbedded remote entry not reachable after ${maxAttempts} attempts ` + + `(${elapsed}s, last status=${resp.status})`, + ); + } + + cy.log( + `remote entry returned ${resp.status} (attempt ${attempt}/${maxAttempts}, ${elapsed}s)`, + ); + // eslint-disable-next-line cypress/no-unnecessary-waiting + return cy.wait(pollIntervalMs).then(() => check(attempt + 1)); + }); + }; + + cy.log(`Polling mlflowEmbedded remote entry (max ${(maxAttempts * pollIntervalMs) / 1000}s)`); + return check(); +}; + +/** + * Enable MLflow backend resources without waiting for sidebar visibility. + * Sets MLflow operator to Managed, waits for it, creates MLflow CR, + * waits for CR readiness and tracking server pod availability. + * + * Useful for composition when a caller will perform its own sidebar check. + */ +const enableMlflowBackend = (): Cypress.Chainable => { const namespace = getApplicationsNamespace(); - cy.step('Enable Gen AI features (required for Prompts nav)'); - return enableGenAiFeatures() - .then(() => { - cy.step('Set MLflow operator to Managed'); - return setMlflowOperatorState('Managed'); - }) + cy.step('Set MLflow operator to Managed'); + return setMlflowOperatorState('Managed') .then(() => { cy.step('Wait for MLflow operator to be ready'); return waitForMlflowOperatorReady(); @@ -209,26 +476,318 @@ export const enablePromptManagementFeatures = (): Cypress.Chainable => return waitForMlflowCRReady(namespace); }) .then(() => { - cy.step('Wait for Prompts nav item in sidebar'); - return waitForPromptsInSidebar(); + cy.step('Wait for MLflow tracking server deployment to be available'); + return waitForMlflowTrackingPodReady(namespace); }); }; /** - * Disable MLflow and Gen AI features. - * Sets mlflowoperator to Removed and disables Gen AI features. + * Enable MLflow operator and tracking server (no Gen AI): + * 1. Set mlflowoperator to Managed and wait for it + * 2. Create an MLflow CR and wait for it to be ready + * 3. Verify DSC status reflects mlflowoperator as Managed + * 4. Establish browser session and verify federated remote is loadable + * 5. Wait for Experiments (MLflow) nav item in the sidebar */ -export const disablePromptManagementFeatures = (): Cypress.Chainable => { +export const enableMlflowFeatures = (): Cypress.Chainable => { + return enableMlflowBackend() + .then(() => { + cy.step('Verify DSC status reflects mlflowoperator as Managed'); + return waitForDSCComponentsManaged(['mlflowoperator']); + }) + .then(() => { + cy.step('Establish browser session for remote entry check'); + cy.visitWithLogin('/'); + return cy.get('#page-sidebar', { timeout: 15000 }); + }) + .then(() => { + cy.step('Wait for mlflowEmbedded module federation remote to be loadable'); + return waitForMlflowRemoteEntry(); + }) + .then(() => { + cy.step('Wait for dashboard backend to reflect mlflowoperator as Managed'); + return waitForDashboardDSCStatus({ mlflowoperator: 'Managed' }); + }) + .then(() => { + cy.step('Wait for Experiments (MLflow) nav item in sidebar'); + return waitForNavItemInSidebar('Experiments (MLflow)'); + }); +}; + +/** + * Restore MLflow to its pre-test state. + * + * @param operatorWasManaged - If true, the operator was already Managed before the test; skip setting it to Removed. + * @param crExisted - If true, the MLflow CR already existed before the test; skip deleting it. + */ +export const disableMlflowFeatures = ( + operatorWasManaged = true, + crExisted = true, +): Cypress.Chainable => { const namespace = getApplicationsNamespace(); - cy.step('Delete MLflow CR'); - return deleteMlflowCR(namespace) + return cy .then(() => { - cy.step('Set MLflow operator to Removed'); - return setMlflowOperatorState('Removed'); + if (!crExisted) { + cy.step('Delete MLflow CR (was not present before test)'); + deleteMlflowCR(namespace); + } }) .then(() => { - cy.step('Disable Gen AI features'); - return disableGenAiFeatures(); + if (!operatorWasManaged) { + cy.step('Set MLflow operator to Removed (was not Managed before test)'); + setMlflowOperatorState('Removed'); + } + }); +}; + +/** + * Enable all features required for Prompt Management: + * 1. Enable Gen AI backend (LlamaStack operator, genAiStudio flag) + * 2. Enable MLflow backend (operator, CR, tracking pod readiness) + * 3. Verify DSC status reflects both operators as Managed + * 4. Single sidebar poll for "Prompts" nav item (confirms frontend picked up the state) + * + * Polls backend resources first so the sidebar check is a quick confirmation, + * not a long discovery loop. + */ +export const enablePromptManagementFeatures = (): Cypress.Chainable => { + cy.step('Enable Gen AI backend (required for Prompts nav)'); + return enableGenAiBackend() + .then(() => { + cy.step('Enable MLflow backend (required for Prompts nav)'); + return enableMlflowBackend(); + }) + .then(() => { + cy.step('Verify DSC status reflects both operators as Managed'); + return waitForDSCComponentsManaged(['llamastackoperator', 'mlflowoperator']); + }) + .then(() => { + cy.step('Establish browser session for remote entry check'); + cy.visitWithLogin('/'); + return cy.get('#page-sidebar', { timeout: 15000 }); + }) + .then(() => { + cy.step('Wait for mlflowEmbedded module federation remote to be loadable'); + return waitForMlflowRemoteEntry(); + }) + .then(() => { + cy.step('Wait for dashboard backend to reflect both operators as Managed'); + return waitForDashboardDSCStatus({ + llamastackoperator: 'Managed', + mlflowoperator: 'Managed', + }); + }) + .then(() => { + cy.step('Wait for Prompts nav item in sidebar'); + return waitForNavItemInSidebar('Prompts'); + }); +}; + +/** + * Restore MLflow and Gen AI features to their pre-test state. + * + * @param operatorWasManaged - If true, the operator was already Managed before the test. + * @param crExisted - If true, the MLflow CR already existed before the test. + * @param genAiWasEnabled - If true, Gen AI features were already enabled before the test. + */ +export const disablePromptManagementFeatures = ( + operatorWasManaged = true, + crExisted = true, + genAiWasEnabled = true, +): Cypress.Chainable => + disableMlflowFeatures(operatorWasManaged, crExisted).then(() => { + if (!genAiWasEnabled) { + cy.step('Disable Gen AI features (were not enabled before test)'); + disableGenAiFeatures(); + } + }); + +/** + * Get the MLflow tracking server URL from the MLflow CR status. + */ +export const getMlflowTrackingUrl = (): Cypress.Chainable => { + const namespace = getApplicationsNamespace(); + return cy + .exec( + `oc get mlflows.mlflow.opendatahub.io -n ${namespace} -o jsonpath="{.items[0].status.address.url}"`, + ) + .then((result) => { + const url = result.stdout.trim().replace(/"/g, ''); + if (!url) { + throw new Error('MLflow tracking URL not found in CR status'); + } + return url; }); }; + +/** + * Get the name of a running MLflow tracking-server pod in the applications namespace. + * Uses the tracking-server label to avoid matching operator, database, or minio pods. + */ +const getMlflowPodName = (): Cypress.Chainable => { + const namespace = getApplicationsNamespace(); + return cy + .exec( + `oc get pods -n ${namespace} -l app=mlflow -o jsonpath="{.items[0].metadata.name}" --field-selector=status.phase=Running`, + { failOnNonZeroExit: false }, + ) + .then((result) => { + const podName = result.stdout.replace(/"/g, '').trim(); + if (!podName) { + throw new Error( + `No running MLflow tracking pod found (label app=mlflow) in namespace ${namespace}`, + ); + } + return podName; + }); +}; + +/** + * Execute a curl command inside the MLflow pod against the tracking server. + */ +const execCurlInMlflowPod = ( + endpoint: string, + body: object, + workspace: string, +): Cypress.Chainable => { + const namespace = getApplicationsNamespace(); + const bodyJson = JSON.stringify(body); + const escapedBody = bodyJson.replace(/'/g, "'\\''"); + return getMlflowPodName().then((podName) => { + const ns = assertNamespace(workspace); + const cmd = [ + `printf '%s' '${escapedBody}'`, + '|', + `oc exec -n ${namespace} -i ${podName} -c mlflow --`, + `curl -sk -X POST 'https://localhost:8443${endpoint}'`, + `-H 'Content-Type: application/json'`, + `-H "Authorization: Bearer $(oc whoami -t)"`, + `-H 'X-MLFLOW-WORKSPACE: ${ns}'`, + `--data-binary @-`, + ].join(' '); + return cy.exec(cmd, { timeout: 30000, log: false }).then((result) => result.stdout.trim()); + }); +}; + +/** + * Create an MLflow experiment via the tracking server REST API. + * + * @param workspace - Namespace to scope the experiment to. + * @param experimentName - Display name for the experiment. + * @returns The experiment_id of the created experiment. + */ +export const createMlflowExperimentViaAPI = ( + workspace: string, + experimentName: string, +): Cypress.Chainable => + execCurlInMlflowPod( + '/api/2.0/mlflow/experiments/create', + { name: experimentName }, + workspace, + ).then((response) => JSON.parse(response).experiment_id); + +/** + * Log a run with parameters and metrics via the MLflow tracking server REST API. + * + * @param workspace - Namespace to scope the run to. + * @param experimentId - The experiment_id to create the run under. + * @param run - Run data (name, parameters, metrics). + * @returns The run_id of the created run. + */ +export const logMlflowRunViaAPI = ( + workspace: string, + experimentId: string, + run: MlflowExperimentRunData, +): Cypress.Chainable => + execCurlInMlflowPod( + '/api/2.0/mlflow/runs/create', + /* eslint-disable camelcase */ + { experiment_id: experimentId, run_name: run.name }, + workspace, + ).then((response) => { + const runId = JSON.parse(response).run.info.run_id; + const params = Object.entries(run.parameters).map(([key, value]) => ({ key, value })); + const metricTimestamp = Date.now(); + const metrics = Object.entries(run.metrics).map(([key, value]) => ({ + key, + value: parseFloat(value), + timestamp: metricTimestamp, + step: 0, + })); + return execCurlInMlflowPod( + '/api/2.0/mlflow/runs/log-batch', + { run_id: runId, params, metrics }, + workspace, + ).then(() => + execCurlInMlflowPod( + '/api/2.0/mlflow/runs/update', + { run_id: runId, status: 'FINISHED', end_time: Date.now() }, + workspace, + ).then(() => runId), + ); + /* eslint-enable camelcase */ + }); + +/** + * Delete an MLflow experiment via the tracking server REST API. + * + * @param workspace - Namespace the experiment is scoped to. + * @param experimentId - The experiment_id to delete. + */ +export const deleteMlflowExperimentViaAPI = ( + workspace: string, + experimentId: string, +): Cypress.Chainable => + execCurlInMlflowPod( + '/api/2.0/mlflow/experiments/delete', + { experiment_id: experimentId }, // eslint-disable-line camelcase + workspace, + ); + +/** + * Look up an MLflow experiment by name and return its ID, or undefined if not found. + */ +export const getMlflowExperimentIdByName = ( + workspace: string, + experimentName: string, +): Cypress.Chainable => + execCurlInMlflowPod( + '/api/2.0/mlflow/experiments/get-by-name', + { experiment_name: experimentName }, // eslint-disable-line camelcase + workspace, + ).then((response) => { + try { + return JSON.parse(response)?.experiment?.experiment_id; + } catch { + return undefined; + } + }); + +/** + * Log multiple runs sequentially under a single experiment. + * + * @param workspace - Namespace to scope the runs to. + * @param experimentId - The experiment_id to create the runs under. + * @param runs - Array of run data objects (name, parameters, metrics). + * @returns Array of run_ids for all created runs. + */ +export const logMlflowRunsViaAPI = ( + workspace: string, + experimentId: string, + runs: MlflowExperimentRunData[], +): Cypress.Chainable => { + const logNext = ( + remaining: MlflowExperimentRunData[], + collected: string[], + ): Cypress.Chainable => { + if (remaining.length === 0) { + return cy.wrap(collected); + } + const [run, ...rest] = remaining; + return logMlflowRunViaAPI(workspace, experimentId, run).then((runId) => + logNext(rest, [...collected, runId]), + ); + }; + return logNext(runs, []); +};