diff --git a/src/app/.gitignore b/src/app/.gitignore index a849c853e..8d047036f 100644 --- a/src/app/.gitignore +++ b/src/app/.gitignore @@ -39,4 +39,7 @@ pnpm-lock.yaml # Temporary files *.tmp -.temp/ \ No newline at end of file +.temp/ + +# Playwright test results +test-results/ diff --git a/src/app/README.md b/src/app/README.md index a4474a210..21de6fe36 100644 --- a/src/app/README.md +++ b/src/app/README.md @@ -54,13 +54,16 @@ npm test Playwright starts an isolated, non-HMR test server on **127.0.0.1:4173** by default. This intentionally avoids reusing a manual development server on port 8080. Set `PLAYWRIGHT_BASE_URL` only when you explicitly want to test an already running server. +> [!NOTE] +> Direct execution via `npx playwright test` runs the accessibility (`a11y`) and `functional` test suites concurrently, which can cause state collisions in local storage/server state. For isolated, sequential execution of the suites, always use `npm test` or `npm run test:headed`. + Runs all UI-safe Playwright tests headless via Chromium. Real-server smoke tests are opt-in so they fail fast instead of silently passing without a running server or loaded model: ```bash LEMONADE_REAL_SERVER=1 npm test ``` -Artifacts are saved under Playwright's per-test output folders, so screenshots from repeated runs are not overwritten. +Artifacts are saved under distinct output folders per project (`test-results/a11y` and `test-results/functional`), preventing results from being overwritten when running the sequential test suites. ### Headed tests (visible browser) diff --git a/src/app/package.json b/src/app/package.json index c50e9fad4..0b36fec97 100644 --- a/src/app/package.json +++ b/src/app/package.json @@ -15,8 +15,8 @@ "build:tauri": "tauri build", "build:nobundle": "tauri build --no-bundle", "build:mac": "tauri build --bundles app", - "test": "npx playwright test", - "test:headed": "npx playwright test --headed", + "test": "node scripts/run-playwright-tests.mjs", + "test:headed": "node scripts/run-playwright-tests.mjs --headed", "test:a11y": "npx playwright test tests/a11y.spec.ts", "test:preset-intent": "node tests/preset-intent.runtime.cjs", "test:autoopt-glue": "node tests/autoopt-glue.runtime.cjs", diff --git a/src/app/playwright.config.ts b/src/app/playwright.config.ts index d714d612c..7e0dfe7f8 100644 --- a/src/app/playwright.config.ts +++ b/src/app/playwright.config.ts @@ -42,7 +42,22 @@ export default defineConfig({ }, projects: [ { - name: 'chromium', + name: 'a11y', + testMatch: '**/a11y.spec.ts', + fullyParallel: true, + outputDir: './test-results/a11y', + use: { + browserName: 'chromium', + viewport: { width: 1280, height: 800 }, + ...(launchOptions ? { launchOptions } : {}), + }, + }, + { + name: 'functional', + testIgnore: '**/a11y.spec.ts', + fullyParallel: false, + workers: 1, + outputDir: './test-results/functional', use: { browserName: 'chromium', viewport: { width: 1280, height: 800 }, diff --git a/src/app/scripts/run-playwright-tests.mjs b/src/app/scripts/run-playwright-tests.mjs new file mode 100755 index 000000000..0f6cf20f2 --- /dev/null +++ b/src/app/scripts/run-playwright-tests.mjs @@ -0,0 +1,52 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import process from 'node:process'; + +const args = process.argv.slice(2); + +// If user explicitly specifies a project, we forward all args directly to a single Playwright run +const hasProject = args.some(arg => arg.startsWith('--project') || arg === '-p'); +const hasHelp = args.some(arg => arg === '--help' || arg === '-h' || arg === 'help'); + +const npxCommand = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + +if (hasProject || hasHelp) { + const result = spawnSync(npxCommand, ['playwright', 'test', ...args], { + stdio: 'inherit', + shell: false, + }); + if (result.error) { + console.error('Failed to start Playwright tests:', result.error.message || result.error); + process.exit(1); + } + process.exit(result.status ?? 1); +} + +console.log('Running accessibility (a11y) tests...'); +const a11yResult = spawnSync(npxCommand, ['playwright', 'test', '--project=a11y', ...args], { + stdio: 'inherit', + shell: false, +}); + +if (a11yResult.error || a11yResult.status !== 0) { + if (a11yResult.error) { + console.error('Failed to start accessibility tests:', a11yResult.error.message || a11yResult.error); + } + process.exit(a11yResult.status ?? 1); +} + +console.log('Running functional tests...'); +const functionalResult = spawnSync(npxCommand, ['playwright', 'test', '--project=functional', ...args], { + stdio: 'inherit', + shell: false, +}); + +if (functionalResult.error || functionalResult.status !== 0) { + if (functionalResult.error) { + console.error('Failed to start functional tests:', functionalResult.error.message || functionalResult.error); + } + process.exit(functionalResult.status ?? 1); +} + +process.exit(0); diff --git a/src/app/tests/a11y.spec.ts b/src/app/tests/a11y.spec.ts index 7dab0ff26..a205683f2 100644 --- a/src/app/tests/a11y.spec.ts +++ b/src/app/tests/a11y.spec.ts @@ -23,6 +23,8 @@ import { test, expect, Page } from '@playwright/test'; import AxeBuilder from '@axe-core/playwright'; +test.describe.configure({ mode: 'parallel' }); + // ─── Constants & helpers ────────────────────────────────────────────────────── const WCAG_TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'] as const; diff --git a/src/app/tests/download-manager-order.spec.ts b/src/app/tests/download-manager-order.spec.ts index bf1994b28..c8d7df32c 100644 --- a/src/app/tests/download-manager-order.spec.ts +++ b/src/app/tests/download-manager-order.spec.ts @@ -54,10 +54,10 @@ test('download manager keeps creation order while concurrent progress updates ar }, ]; - await page.route('/api/v1/health', route => + await page.route('**/api/v1/health**', route => route.fulfill({ json: { status: 'ok', all_models_loaded: [] } }), ); - await page.route('/api/v1/downloads**', route => route.fulfill({ json: { downloads: serverDownloads } })); + await page.route('**/api/v1/downloads**', route => route.fulfill({ json: { downloads: serverDownloads } })); await page.goto('/'); await page.locator('.titlebar__download-toggle').click(); @@ -154,10 +154,10 @@ test('authoritative server snapshot removes stale paused renderer state', async updatedAt: timestamp, }); - await page.route('/api/v1/health', route => + await page.route('**/api/v1/health**', route => route.fulfill({ json: { status: 'ok', all_models_loaded: [] } }), ); - await page.route('/api/v1/downloads**', route => route.fulfill({ json: { downloads: [] } })); + await page.route('**/api/v1/downloads**', route => route.fulfill({ json: { downloads: [] } })); await page.goto('/'); await page.locator('.titlebar__download-toggle').click();