WIP - Add experimental Firefox integration test support

Warning: These changes were initially generated with Cursor, and I am now in the
         process of tidying them up.

Author: kzar

.github/workflows/checks.yml

@@ -67,7 +67,34 @@ jobs:
             - uses: actions/upload-artifact@v4
               if: always()
               with:
-                  name: ${{ matrix.test }}-report
+                  name: ${{ matrix.test }}-report-${{ strategy.job-index }}
                   path: playwright-report/
                   retention-days: 1
-                  
+
+    playwright-tests-firefox:
+        timeout-minutes: 30
+        runs-on: ubuntu-latest
+        # Firefox tests are experimental - don't block CI on failures
+        continue-on-error: true
+        strategy:
+            fail-fast: false
+            matrix:
+                shard: [1/4, 2/4, 3/4, 4/4]
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: 22
+                  cache: 'npm'
+            - run: npm run install-ci
+            - name: Install Playwright Browsers
+              run: npx playwright install --with-deps firefox
+            - name: Run Firefox tests
+              run: xvfb-run --auto-servernum -- npm run playwright-firefox -- --shard ${{ matrix.shard }}
+            - uses: actions/upload-artifact@v4
+              if: always()
+              with:
+                  name: playwright-firefox-report-${{ strategy.job-index }}
+                  path: playwright-report/
+                  retention-days: 1
+

integration-test/README.md

@@ -12,11 +12,18 @@ We use [Playwright](https://playwright.dev/) as a test runner. Tests are defined
 
 Playwright tests can be run with the following npm commands:
  - `npm run playwright` to test the Chrome MV3 extension
- - `npm run playwright-mv2` to test the Chrome MV2 extension (closest thing to testing Firefox MV2 extension we have until Playwright adds support for testing Firefox extensions)
+ - `npm run playwright-mv2` to test the Chrome MV2 extension
 
 If you want to re-run tests without rebuilding the extension, you can subsequently run:
  - `npx playwright test` to run all tests
- - `npx playright test integration-test/<file>.spec.js` to just run tests in a single file.
+ - `npx playwright test integration-test/<file>.spec.js` to just run tests in a single file.
+
+### Firefox Testing (Experimental)
+
+Firefox integration tests are available for a subset of tests. The Firefox harness uses the
+Remote Debugging Protocol (RDP) to install the extension and evaluate code in its background page.
+So far many tests can't run because request interception is not working correctly.
+ - `npm run playwright-firefox`
 
 ## Writing tests
 
@@ -40,10 +47,10 @@ test('my test', async ({ manifestVersion, page, backgroundPage, backgroundNetwor
 The arguments to the test function are:
  - `manifestVersion`: `2` or `3`. Allows you to check which version of the extension is being tested.
  - `page`: A [Page](https://playwright.dev/docs/api/class-page) instance for loading web pages.
- - `backgroundPage`: The extension's background page, which is a `Page` for MV2, or `Worker` for MV3. Use `backgroundPage.evaluate` to run code in the extension's background context.
+ - `backgroundPage`: The extension's background page. For Chrome MV2, this is a `Page`. For Chrome MV3, this is a `Worker`. For Firefox, this is a `FirefoxBackgroundPage` wrapper that supports `evaluate()` via RDP. Use `backgroundPage.evaluate()` to run code in the extension's background context.
  - `backgroundNetworkContext`: A context for listening to and intercepting requests from the extension's background context with Playwright's [Network](https://playwright.dev/docs/network) APIs. 
  - `context`: The [BrowserContext](https://playwright.dev/docs/api/class-browsercontext) for the test run.
 
 Static files for tests are in the `data` directory:
  - `har` - [HAR files](./data/har/README.md) for offline tests.
- - `staticcdn` - Mocked CDN resources to used when the extension loads.
+ - `staticcdn` - Mocked CDN resources to used when the extension loads.

integration-test/background-eval.spec.js

@@ -0,0 +1,50 @@
+import { test, expect, isFirefox } from './helpers/playwrightHarness.js';
+import backgroundWait from './helpers/backgroundWait';
+
+test.describe('test harness background page/ServiceWorker evaluation', () => {
+    test('can evaluate code in extension background', async ({ context, backgroundPage }) => {
+        expect(backgroundPage).not.toBeNull();
+        await backgroundWait.forExtensionLoaded(context);
+
+        if (isFirefox()) {
+            expect(backgroundPage.isAvailable()).toBe(true);
+        } else {
+            expect(backgroundPage.isAvailable).toBe(undefined);
+        }
+
+        // Test simple evaluation
+        const manifestName = await backgroundPage.evaluate(() => {
+            return chrome.runtime.getManifest().name;
+        });
+        expect(manifestName.startsWith('DuckDuckGo')).toBe(true);
+
+        // Test evaluation with arguments
+        const result = await backgroundPage.evaluate(
+            ({ a, b }) => {
+                return a + b;
+            },
+            { a: 2, b: 3 },
+        );
+        expect(result).toBe(5);
+
+        // Test string argument
+        const greeting = await backgroundPage.evaluate((name) => {
+            return 'Hello, ' + name;
+        }, 'Firefox');
+        expect(greeting).toBe('Hello, Firefox');
+
+        // Test returning an object (important for compatibility with existing tests)
+        const obj = await backgroundPage.evaluate(() => {
+            return { foo: 'bar', num: 42, bool: true };
+        });
+        expect(obj.foo).toBe('bar');
+        expect(obj.num).toBe(42);
+        expect(obj.bool).toBe(true);
+
+        // Test returning an array
+        const arr = await backgroundPage.evaluate(() => {
+            return [1, 2, 3, 'four'];
+        });
+        expect(arr).toEqual([1, 2, 3, 'four']);
+    });
+});

integration-test/fingerprint-protection.spec.js

@@ -1,4 +1,4 @@
-import { test, expect, getHARPath } from './helpers/playwrightHarness';
+import { test, expect, getHARPath, addScriptTag } from './helpers/playwrightHarness';
 import backgroundWait from './helpers/backgroundWait';
 import { overridePrivacyConfig } from './helpers/testConfig';
 
@@ -61,7 +61,7 @@ test.describe('First Party Fingerprint Randomization', () => {
     async function runTest(testCase, page) {
         await page.routeFromHAR(testCase.har);
         await page.goto(`https://${testCase.url}`);
-        await page.addScriptTag({ path: 'node_modules/@fingerprintjs/fingerprintjs/dist/fp.js' });
+        await addScriptTag(page, { path: 'node_modules/@fingerprintjs/fingerprintjs/dist/fp.js' });
 
         const fingerprint = await page.evaluate(() => {
             /* global FingerprintJS */

integration-test/fingerprint-randomization.spec.js

@@ -1,13 +1,13 @@
 import fs from 'fs';
 import path from 'path';
 
-import { test, expect } from './helpers/playwrightHarness';
+import { test, expect, addScriptTag } from './helpers/playwrightHarness';
 import { forExtensionLoaded } from './helpers/backgroundWait';
 
 const fakeOrigin = 'http://test.example';
 
 async function getFingerprintOfContext(ctx) {
-    await ctx.addScriptTag({ path: 'node_modules/@fingerprintjs/fingerprintjs/dist/fp.js' });
+    await addScriptTag(ctx, { path: 'node_modules/@fingerprintjs/fingerprintjs/dist/fp.js' });
     return ctx.evaluate(() => {
         /* global FingerprintJS */
         return (async () => {

integration-test/helpers/backgroundWait.js

@@ -9,13 +9,13 @@ function manuallyWaitForFunction(bgPage, func, { polling, timeout }, ...args) {
             try {
                 result = await bgPage.evaluate(func, ...args);
             } catch (e) {
-                reject(e);
+                return reject(e);
             }
             if (result) {
-                resolve(result);
+                return resolve(result);
             } else {
                 if (Date.now() - startTime > timeout) {
-                    reject(new errors.TimeoutError('Manually waiting for function timed out: ' + func.toString()));
+                    return reject(new errors.TimeoutError('Manually waiting for function timed out: ' + func.toString()));
                 } else {
                     setTimeout(waitForFunction, polling);
                 }