feat(FR-2877): implement webui-smoke-cli runner with tag filter and external endpoint

Author: yomybaby

packages/backend.ai-webui-smoke-cli/README.md

@@ -4,3 +4,15 @@ Post-install smoke verification CLI for Backend.AI WebUI. See
 [`.specs/FR-2871-webui-smoke-cli/spec.md`](../../.specs/FR-2871-webui-smoke-cli/spec.md)
 for the full spec. The operator-facing README (EN + KO) lands in FR-2883
 (Phase 2).
+
+## Usage (alpha)
+
+```sh
+bai-smoke run \
+  --endpoint https://webui.example.com \
+  --email admin@example.com \
+  --password "***" \
+  --output ./smoke-report
+```
+
+This is an alpha MVP. Full operator docs land with FR-2883.

packages/backend.ai-webui-smoke-cli/package.json

@@ -19,6 +19,7 @@
   "files": [
     "bin",
     "dist",
+    "playwright.smoke.config.ts",
     "README.md"
   ],
   "scripts": {
@@ -30,6 +31,7 @@
     "version:print": "node ./bin/bai-smoke.cjs version"
   },
   "dependencies": {
+    "@playwright/test": "^1.58.2",
     "commander": "^12.1.0"
   },
   "devDependencies": {

packages/backend.ai-webui-smoke-cli/playwright.smoke.config.ts

@@ -0,0 +1,86 @@
+import { defineConfig, devices } from '@playwright/test';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+/**
+ * Playwright configuration for the `bai-smoke` runner.
+ *
+ * This config is invoked by `bai-smoke run` (`src/runner.ts`) via
+ * `npx playwright test --config <this-file>`. The runner is the single
+ * place that translates CLI arguments into the env vars consumed here —
+ * we deliberately do NOT load `e2e/envs/.env.playwright` from this file
+ * because the smoke CLI runs against arbitrary customer endpoints, not
+ * the dev fixtures the e2e suite assumes.
+ *
+ * Env contract (set by the runner):
+ *   BAI_SMOKE_REPORT_DIR    — output directory for html / json reports
+ *   BAI_SMOKE_WORKERS       — playwright worker count (optional)
+ *   BAI_SMOKE_TIMEOUT_MS    — per-test timeout in ms (default 180000)
+ *   BAI_SMOKE_GREP          — grep regex source (no slashes), e.g. `@smoke(-any|-admin)?\b`
+ *   BAI_SMOKE_GREP_INVERT   — grepInvert regex source (optional)
+ *   BAI_SMOKE_PAGES         — comma-separated page directory names (optional)
+ *   BAI_SMOKE_HEADED        — '1' to launch a headed browser
+ *
+ * Test credentials (E2E_*) are set by the runner as well — see
+ * `src/config.ts#buildPlaywrightEnv`.
+ */
+
+// Resolve repo root e2e/ directory relative to this config file.
+// File layout: <repo>/packages/backend.ai-webui-smoke-cli/playwright.smoke.config.ts
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const E2E_DIR = path.resolve(__dirname, '..', '..', 'e2e');
+
+const reportDir = process.env.BAI_SMOKE_REPORT_DIR ?? path.resolve(process.cwd(), 'smoke-report');
+
+const workersEnv = process.env.BAI_SMOKE_WORKERS;
+const workers = workersEnv ? Number.parseInt(workersEnv, 10) : undefined;
+
+const timeoutEnv = process.env.BAI_SMOKE_TIMEOUT_MS;
+const timeout = timeoutEnv ? Number.parseInt(timeoutEnv, 10) : 180000;
+
+const grepSource = process.env.BAI_SMOKE_GREP;
+const grepInvertSource = process.env.BAI_SMOKE_GREP_INVERT;
+
+// Pages filter — match by directory under e2e/. e.g. BAI_SMOKE_PAGES="session,vfolder"
+// turns into testMatch ['**/session/**', '**/vfolder/**'].
+const pagesEnv = process.env.BAI_SMOKE_PAGES;
+const pages = pagesEnv
+  ? pagesEnv
+      .split(',')
+      .map((p) => p.trim())
+      .filter(Boolean)
+  : undefined;
+const testMatch = pages && pages.length > 0 ? pages.map((p) => `**/${p}/**`) : undefined;
+
+const headed = process.env.BAI_SMOKE_HEADED === '1';
+
+export default defineConfig({
+  testDir: E2E_DIR,
+  testMatch,
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  workers: workers && Number.isFinite(workers) && workers > 0 ? workers : undefined,
+  timeout: Number.isFinite(timeout) && timeout > 0 ? timeout : 180000,
+  grep: grepSource ? new RegExp(grepSource) : undefined,
+  grepInvert: grepInvertSource ? new RegExp(grepInvertSource) : undefined,
+  reporter: [
+    ['html', { outputFolder: path.join(reportDir, 'html'), open: 'never' }],
+    ['json', { outputFile: path.join(reportDir, 'results.json') }],
+    ['list'],
+  ],
+  snapshotPathTemplate: `${E2E_DIR}/{testFileDir}/snapshot/{arg}{ext}`,
+  use: {
+    trace: 'retain-on-failure',
+    video: 'retain-on-failure',
+    headless: !headed,
+    ignoreHTTPSErrors: true,
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'], locale: 'en-US' },
+    },
+  ],
+});

packages/backend.ai-webui-smoke-cli/src/cli.ts

@@ -1,18 +1,24 @@
 /**
  * Entrypoint for the `bai-smoke` CLI.
  *
- * MVP scaffold (FR-2876):
- * - `list`    — print the catalog of smoke categories. Functional.
- * - `version` — print CLI version + bundled SHA + Playwright version. Functional.
- * - `run`     — stub. The actual Playwright runner ships in FR-2877.
+ * Subcommands (FR-2877 MVP):
+ *   - `list`    — print the catalog of smoke categories.
+ *   - `version` — print CLI + WebUI + Playwright version info.
+ *   - `run`     — execute the smoke suite against a customer endpoint.
  *
- * Global option parsing (`--endpoint`, `--email`, etc.) is declared on `run`
- * so `--help` documents the eventual flag surface, but every flag is a no-op
- * at this stage.
+ * Phase 2 subcommands (`doctor`, `preflight`) ship under FR-2878+.
  */
-import { Command } from 'commander';
+import { Command, Option } from 'commander';
+import path from 'node:path';
 
 import { SMOKE_CATALOG } from './catalog.js';
+import {
+  parseDuration,
+  splitCsvArg,
+  type SmokeRoleSelection,
+  type SmokeRunOptions,
+} from './config.js';
+import { runSmoke } from './runner.js';
 import {
   CLI_NAME,
   CLI_VERSION,
@@ -61,39 +67,147 @@ program
     process.stdout.write(`  platform           : ${process.platform}-${process.arch}\n`);
   });
 
-// `run` is a stub until FR-2877 wires the Playwright runner.
 program
   .command('run')
-  .description(
-    'Run the smoke suite against an endpoint. Not yet implemented — coming in FR-2877.',
+  .description('Run the smoke suite against a Backend.AI WebUI endpoint.')
+  .addOption(
+    new Option('--endpoint <url>', 'Backend.AI WebUI endpoint URL.')
+      .env('BAI_SMOKE_ENDPOINT')
+      .makeOptionMandatory(true),
+  )
+  .option(
+    '--webserver <url>',
+    'Backend.AI webserver endpoint URL. Defaults to --endpoint when omitted.',
+  )
+  .addOption(
+    new Option('--email <email>', 'Login email or username.')
+      .env('BAI_SMOKE_EMAIL')
+      .makeOptionMandatory(true),
+  )
+  .addOption(
+    new Option(
+      '--password <password>',
+      'Login password. Prefer --password-stdin or BAI_SMOKE_PASSWORD env.',
+    ).env('BAI_SMOKE_PASSWORD'),
+  )
+  .option('--password-stdin', 'Read the password from stdin instead of --password.')
+  .addOption(
+    new Option('--role <role>', 'Role selection: auto, admin, or user.')
+      .choices(['auto', 'admin', 'user'])
+      .default('auto'),
   )
-  .option('--endpoint <url>', 'Backend.AI webui endpoint URL.')
-  .option('--webserver <url>', 'Backend.AI webserver endpoint URL.')
-  .option('--email <email>', 'Account email or username.')
-  .option('--password <password>', 'Account password (prefer --password-stdin).')
-  .option('--password-stdin', 'Read the password from stdin.')
+  .option('--include <tags>', 'Comma-separated extra tags to include (e.g. "@critical").')
+  .option('--exclude <tags>', 'Comma-separated tags to exclude.')
   .option(
-    '--role <role>',
-    'Force role selection: auto | admin | user | monitor.',
-    'auto',
+    '--pages <names>',
+    'Comma-separated page directory names (e.g. "session,vfolder").',
+  )
+  .option('--workers <n>', 'Playwright worker count.', (v) => Number.parseInt(v, 10))
+  .option(
+    '--timeout <duration>',
+    'Per-test timeout. Accepts "180s", "3m", or raw ms.',
+    '180s',
+  )
+  .option(
+    '--output <dir>',
+    'Output directory for the smoke report.',
+    () => defaultOutputDir(),
+    defaultOutputDir(),
   )
-  .option('--include <tags...>', 'Additional tags to include in the run.')
-  .option('--exclude <tags...>', 'Tags to exclude from the run.')
-  .option('--pages <pages...>', 'Restrict the run to specific page categories.')
-  .option('--workers <n>', 'Playwright worker count.', '1')
-  .option('--timeout <ms>', 'Per-test timeout in milliseconds.', '120000')
-  .option('--output <dir>', 'Output directory for the smoke report.', './smoke-report')
   .option('--headed', 'Run the browser in headed mode (debugging only).', false)
   .option('--insecure-tls', 'Accept self-signed TLS certificates.', false)
-  .action(() => {
-    process.stderr.write(
-      'bai-smoke run: Not yet implemented. Coming in FR-2877.\n',
-    );
-    process.exit(2);
+  .action(async (raw: Record<string, unknown>) => {
+    const password = await resolvePassword(raw);
+    if (!password) {
+      process.stderr.write(
+        'bai-smoke run: --password, --password-stdin, or BAI_SMOKE_PASSWORD is required.\n',
+      );
+      process.exit(2);
+    }
+
+    const endpoint = String(raw.endpoint);
+    const webserver =
+      typeof raw.webserver === 'string' && raw.webserver.length > 0
+        ? raw.webserver
+        : endpoint;
+    if (webserver === endpoint && !raw.webserver) {
+      process.stderr.write(
+        '[bai-smoke] --webserver not supplied; reusing --endpoint as the webserver URL.\n',
+      );
+    }
+
+    let timeoutMs: number;
+    try {
+      timeoutMs = parseDuration(String(raw.timeout ?? '180s'));
+    } catch (err) {
+      process.stderr.write(`bai-smoke run: ${(err as Error).message}\n`);
+      process.exit(2);
+      return;
+    }
+
+    const opts: SmokeRunOptions = {
+      endpoint,
+      webserver,
+      email: String(raw.email),
+      password,
+      role: (raw.role as SmokeRoleSelection) ?? 'auto',
+      include: splitCsvArg(raw.include as string | string[] | undefined),
+      exclude: splitCsvArg(raw.exclude as string | string[] | undefined),
+      pages: splitCsvArg(raw.pages as string | string[] | undefined),
+      workers: typeof raw.workers === 'number' && raw.workers > 0 ? raw.workers : undefined,
+      timeoutMs,
+      outputDir: path.resolve(String(raw.output ?? defaultOutputDir())),
+      headed: raw.headed === true,
+      insecureTls: raw.insecureTls === true,
+    };
+
+    const { exitCode, reportPath, summary } = await runSmoke(opts);
+
+    process.stdout.write('\n');
+    process.stdout.write('bai-smoke summary\n');
+    process.stdout.write(`${'-'.repeat(60)}\n`);
+    process.stdout.write(`  endpoint  : ${summary.endpoint}\n`);
+    process.stdout.write(`  webserver : ${summary.webserver}\n`);
+    process.stdout.write(`  role      : ${summary.role} (selection: ${summary.roleSelection})\n`);
+    if (summary.results) {
+      const { total, passed, failed, skipped, flaky } = summary.results;
+      process.stdout.write(
+        `  results   : ${passed} passed, ${failed} failed, ${skipped} skipped, ${flaky} flaky (total ${total})\n`,
+      );
+    } else {
+      process.stdout.write('  results   : (no JSON reporter output found)\n');
+    }
+    process.stdout.write(`  report    : ${reportPath}\n`);
+    process.stdout.write(`  summary   : ${path.join(opts.outputDir, 'summary.json')}\n`);
+    process.exit(exitCode);
   });
 
 program.parseAsync(process.argv).catch((err: unknown) => {
   // eslint-disable-next-line no-console
   console.error(err);
   process.exit(1);
 });
+
+function defaultOutputDir(): string {
+  const iso = new Date().toISOString().replace(/[:.]/g, '-');
+  return path.resolve(process.cwd(), `smoke-report-${iso}`);
+}
+
+async function resolvePassword(raw: Record<string, unknown>): Promise<string | undefined> {
+  if (typeof raw.password === 'string' && raw.password.length > 0) {
+    return raw.password;
+  }
+  if (raw.passwordStdin === true) {
+    return readStdin();
+  }
+  return undefined;
+}
+
+function readStdin(): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    process.stdin.on('data', (c: Buffer) => chunks.push(c));
+    process.stdin.on('end', () => resolve(Buffer.concat(chunks).toString('utf8').trim()));
+    process.stdin.on('error', reject);
+  });
+}