test: capture test trace and attachments

Author: ruifigueira

packages/playwright-cloudflare/internal.d.ts

@@ -56,6 +56,7 @@ export type TestContext = {
   env: Env;
   sessionId: string;
   assetsUrl: string;
+  retry: number;
 };
 
 export function currentTestContext(): TestContext;

packages/playwright-cloudflare/src/internal.ts

@@ -1,3 +1,5 @@
+import fs from 'fs';
+
 import { asLocator, currentZone, isString, ManualPromise, setTimeOrigin, timeOrigin } from 'playwright-core/lib/utils';
 import { loadConfig } from 'playwright/lib/common/configLoader';
 import { currentTestInfo, setCurrentlyLoadingFileSuite } from 'playwright/lib/common/globals';
@@ -62,7 +64,6 @@ function toInfo(test: Suite | TestCase): SuiteInfo | TestCaseInfo {
 }
 
 export const playwrightTestConfig = {
-  preserveOutput: 'failures-only',
   projects: [
     {
       timeout: 5000,
@@ -115,13 +116,21 @@ class TestWorker extends WorkerMain {
 
   protected override dispatchEvent(method: string, params: any): void {
     if (method === 'attach') {
-      const { name, body, contentType } = params;
-      if (!body)
-        return;
-      this._attachments.push({ name, body, contentType });
+      const { name, body, path, contentType } = params;
+      let fileContent: string | undefined;
+      if (!body) {
+        if (!path)
+          throw new Error('Either body or path must be provided');
+        if (!fs.existsSync(path))
+          throw new Error(`File does not exist: ${path}`);
+        fileContent = fs.readFileSync(path, 'base64') as string;
+      }
+      this._attachments.push({ name, body: body ?? fileContent, contentType });
     }
+
     if (method === 'testEnd')
       this._testResult = params;
+
     if (method === 'done') {
       if (!this._testResult) {
         this._testResult = {
@@ -170,9 +179,10 @@ export class TestRunner {
     context = this._testContext;
     const testWorker = new TestWorker(this._options);
     try {
+      const { retry } = this._testContext;
       const [result] = await Promise.all([
         testWorker.testResult(),
-        testWorker.runTestGroup({ file, entries: [{ testId, retry: 0 }] }),
+        testWorker.runTestGroup({ file, entries: [{ testId, retry }] }),
       ]);
       if (result.status === 'failed' && result.errors.some(isUnsupportedOperationError)) {
         return {
@@ -223,6 +233,7 @@ function renderApiCall(apiName: string, params: any) {
 }
 
 const tracingGroupSteps: TestStepInternal[] = [];
+// adapted from _setupArtifacts fixture in packages/playwright/src/index.ts
 const expectApiListener: ClientInstrumentationListener = {
   onApiCallBegin: (data: ApiCallData) => {
     const testInfo = currentTestInfo();

packages/playwright-cloudflare/tests/playwright.config.ts

@@ -5,6 +5,7 @@ export default defineConfig({
   reporter: process.env.CI ? 'dot' : 'list',
   // dev mode apparently doesn't support parallelism
   workers: process.env.TESTS_SERVER_URL ? 6 : 1,
+  retries: process.env.TESTS_SERVER_URL ? 1 : 0,
   timeout: 60 * 1000,
   projects: [
     {

packages/playwright-cloudflare/tests/src/proxy/proxyTests.ts

@@ -66,7 +66,7 @@ export async function proxyTests(file: string) {
     },
 
     runTest: async ({ testId, fullTitle }: { testId: string, fullTitle: string }, testInfo: TestInfo) => {
-      const response = await fetch(url, { body: JSON.stringify({ testId, fullTitle }), method: 'POST' });
+      const response = await fetch(url, { body: JSON.stringify({ testId, fullTitle, retry: testInfo.retry }), method: 'POST' });
       if (!response.ok)
         throw new Error(`Failed to run test ${fullTitle} (${testId})`);
 

packages/playwright-cloudflare/tests/src/server/testsServer.ts

@@ -8,6 +8,7 @@ export type TestRequestPayload = {
   testId: string;
   fullTitle: string;
   timeout: number;
+  retry: number;
 };
 
 // eslint-disable-next-line no-console
@@ -30,10 +31,11 @@ export class TestsServer extends DurableObject<Env> {
     if (!sessionId)
       return new Response('sessionId is required', { status: 400 });
     const timeout = parseInt(url.searchParams.get('timeout') ?? '10', 10) * 1000;
-    const { testId, fullTitle } = await request.json() as TestRequestPayload;
+    const { testId, fullTitle, retry } = await request.json() as TestRequestPayload;
     const assetsUrl = url.origin;
     const { env } = this;
-    const testRunner = new TestRunner({ env, sessionId, assetsUrl }, { timeout });
+    const context = { env, sessionId, assetsUrl, retry };
+    const testRunner = new TestRunner(context, { timeout });
     if (skipTestsFullTitles.has(fullTitle)) {
       log(`🚫 Skipping ${fullTitle}`);
       return Response.json({
@@ -48,7 +50,7 @@ export class TestsServer extends DurableObject<Env> {
       } satisfies TestEndPayload);
     }
 
-    log(`🧪 Running ${fullTitle}`);
+    log(`🧪 Running ${fullTitle}${retry ? ` (retry #${retry})` : ''}`);
 
     const result = await testRunner.runTest(file, testId);
     if (!['passed', 'skipped'].includes(result.status))

packages/playwright-cloudflare/tests/src/server/workerFixtures.ts

@@ -117,7 +117,12 @@ export type PageWorkerFixtures = {
   headless: boolean;
   channel: string;
   screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick<PageScreenshotOptions, 'fullPage' | 'omitBackground'>;
-  trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'retain-on-first-failure' | 'on-all-retries' | /** deprecated */ 'retry-with-trace';
+  // we can't use trace fixture because plawyright triggers trace start/stop automatically if that fixture is available
+  // and for some reason it causes a timeout, so we implement a simplified version of trace fixture here
+  // See: https://github.com/microsoft/playwright/blob/v1.51.1/packages/playwright/src/worker/workerMain.ts#L342
+  traceMode: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'retain-on-first-failure' | 'on-all-retries';
+  // the official trace fixture must be set to off
+  trace: 'off';
   video: VideoMode | { mode: VideoMode, size: ViewportSize };
   browserName: 'chromium';
   browserVersion: string;
@@ -164,6 +169,7 @@ export const test = platformTest.extend<PageTestFixtures & ServerFixtures & Test
   headless: [true, { scope: 'worker' }],
   channel: ['stable', { scope: 'worker' }],
   screenshot: ['off', { scope: 'worker' }],
+  traceMode: ['on-first-retry', { scope: 'worker' }],
   trace: ['off', { scope: 'worker' }],
   video: ['off', { scope: 'worker' }],
   browserName: ['chromium', { scope: 'worker' }],
@@ -216,15 +222,43 @@ export const test = platformTest.extend<PageTestFixtures & ServerFixtures & Test
     await run(_combinedContextOptions);
   },
 
-  context: async ({ contextFactory, _combinedContextOptions }, run, testInfo) => {
+  context: async ({ contextFactory, _combinedContextOptions, traceMode }, run, testInfo) => {
     const context = await contextFactory(_combinedContextOptions);
-    await context.tracing.start({ screenshots: true, snapshots: true });
+    const traceStarted = traceMode === 'on' ||
+      (traceMode === 'retain-on-failure' && testInfo.retry === 0) ||
+      (traceMode === 'on-first-retry' && testInfo.retry === 1) ||
+      (traceMode === 'on-all-retries' && testInfo.retry > 0);
+    if (traceStarted)
+      await context.tracing.start({ screenshots: true, snapshots: true });
+
     await run(context);
-    if (testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus) {
+
+    const failed = testInfo.status !== testInfo.expectedStatus;
+    let keepTrace = false;
+    if (traceStarted) {
+      switch (traceMode) {
+        case 'on':
+          keepTrace = true;
+          break;
+        case 'on-first-retry':
+          keepTrace = testInfo.retry === 1;
+          break;
+        case 'on-all-retries':
+          keepTrace = testInfo.retry > 0;
+          break;
+        case 'retain-on-failure':
+          keepTrace = failed;
+          break;
+      }
+    }
+
+    if (testInfo.status !== 'skipped' && keepTrace) {
       const tracePath = testInfo.outputPath('trace.zip');
       await context.tracing.stop({ path: tracePath });
-      const trace = await fs.promises.readFile(tracePath);
-      testInfo.attachments.push({ name: 'trace', body: trace, contentType: 'application/zip' });
+      testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });
+    } else if (traceStarted) {
+      // stop and discard trace
+      await context.tracing.stop();
     }
   },