test: Add E2E test for first install

Add E2E test to ensure a window opens upon first install.

This test needs to use a production-like build rather than a standard E2E
build, so the existing "vault decryptor" job was repurposed to be more
generically for E2E tests using a production-like build.

A global function `reloadExtension` is added to `stateHooks` for use by this
test. This function had to be enabled for production builds because the test
uses a production build.

Author: Gudahtt

.circleci/config.yml

@@ -174,14 +174,22 @@ workflows:
       - test-e2e-swap-playwright - OPTIONAL:
           requires:
             - prep-build
-      - test-e2e-chrome-vault-decryption:
+      - test-e2e-chrome-dist:
           filters:
             branches:
               only:
                 - main
                 - /^Version-v(\d+)[.](\d+)[.](\d+)/
           requires:
             - prep-build
+      - test-e2e-firefox-dist:
+          filters:
+            branches:
+              only:
+                - main
+                - /^Version-v(\d+)[.](\d+)[.](\d+)/
+          requires:
+            - prep-build-mv2
       - validate-source-maps:
           requires:
             - prep-build
@@ -213,7 +221,7 @@ workflows:
             - test-e2e-firefox
             - test-e2e-chrome-flask
             - test-e2e-firefox-flask
-            - test-e2e-chrome-vault-decryption
+            - test-e2e-chrome-dist
             - test-e2e-chrome-webpack
       - job-publish-prerelease:
           requires:
@@ -776,17 +784,38 @@ jobs:
       - store_test_results:
           path: test/test-results/e2e
 
-  test-e2e-chrome-vault-decryption:
+  test-e2e-chrome-dist:
     executor: node-browsers-medium-plus
     steps:
       - run: *shallow-git-clone-and-enable-vnc
       - run: sudo corepack enable
       - attach_workspace:
           at: .
       - run:
-          name: test:e2e:single
+          name: Vault decryptor tests
           command: .circleci/scripts/test-run-e2e.sh yarn test:e2e:single test/e2e/vault-decryption-chrome.spec.ts --browser chrome
           no_output_timeout: 5m
+      - run:
+          name: First install tests
+          command: .circleci/scripts/test-run-e2e.sh yarn test:e2e:single test/e2e/first-install.spec.ts --browser chrome
+          no_output_timeout: 5m
+      - store_artifacts:
+          path: test-artifacts
+          destination: test-artifacts
+      - store_test_results:
+          path: test/test-results/e2e
+
+  test-e2e-firefox-dist:
+    executor: node-browsers-medium-plus
+    steps:
+      - run: *shallow-git-clone-and-enable-vnc
+      - run: sudo corepack enable
+      - attach_workspace:
+          at: .
+      - run:
+          name: First install tests
+          command: .circleci/scripts/test-run-e2e.sh yarn test:e2e:single test/e2e/first-install.spec.ts --browser firefox
+          no_output_timeout: 5m
       - store_artifacts:
           path: test-artifacts
           destination: test-artifacts

test/e2e/first-install.spec.ts

@@ -0,0 +1,71 @@
+import assert from 'assert/strict';
+import { Browser } from 'selenium-webdriver';
+import { withFixtures } from './helpers';
+import { errorMessages } from './webdriver/driver';
+import StartOnboardingPage from './page-objects/pages/onboarding/start-onboarding-page';
+import { hasProperty, isObject } from '@metamask/utils';
+
+// Window handle adjustments will need to be made for Non-MV3 Firefox
+// due to OffscreenDocument.
+const IS_FIREFOX = process.env.SELENIUM_BROWSER === Browser.FIREFOX;
+
+// The number of windows to expect to be open
+const baseWindows =
+  1 + // The primary window, starts out on extensions page
+  (IS_FIREFOX ? 0 : 1); // The offscreen document, only on Chrome/MV3
+
+describe('First install', function () {
+  it('opens new window upon install, but not on subsequent reloads', async function () {
+    await withFixtures(
+      {
+        disableServerMochaToBackground: true,
+      },
+      async ({ driver }) => {
+        // Wait for MetaMask to automatically open a new tab
+        await driver.waitUntilXWindowHandles(baseWindows + 1);
+
+        // We cannot use customized driver functions related to window handles, as they depend upon
+        // the background websocket connection enabled only for E2E test builds. This test runs on
+        // a production-like build, which doesn't have this websocket connection.
+        let windowHandles = await driver.driver.getAllWindowHandles();
+
+        // Switch to new tab and verify it's the start onboarding page
+        await driver.driver.switchTo().window(windowHandles[baseWindows]);
+        const startOnboardingPage = new StartOnboardingPage(driver);
+        await startOnboardingPage.check_pageIsLoaded();
+
+        await driver.executeScript('window.stateHooks.reloadExtension()');
+
+        // Wait for extension to reload, signified by the onboarding tab closing
+        await driver.waitUntilXWindowHandles(baseWindows);
+
+        // Test to see if it re-opens
+        try {
+          await driver.waitUntilXWindowHandles(baseWindows + 1);
+        } catch (error) {
+          if (
+            isObject(error) &&
+            hasProperty(error, 'message') &&
+            typeof error.message === 'string' &&
+            error.message.startsWith(
+              errorMessages.waitUntilXWindowHandlesTimeout,
+            )
+          ) {
+            // Ignore timeout error, it's expected here in the success case
+            console.log('Onboarding tab not opened');
+          } else {
+            throw error;
+          }
+        }
+
+        // Ensure new tab is not open
+        windowHandles = await driver.driver.getAllWindowHandles();
+        assert.equal(
+          windowHandles.length,
+          baseWindows,
+          `Expected ${baseWindows} windows, found ${windowHandles.length}`,
+        );
+      },
+    );
+  });
+});

test/e2e/webdriver/driver.js

@@ -23,6 +23,14 @@ const PAGES = {
   POPUP: 'popup',
 };
 
+/**
+ * Error messages used by driver methods.
+ */
+const errorMessages = {
+  waitUntilXWindowHandlesTimeout:
+    'waitUntilXWindowHandles timed out polling window handles',
+};
+
 /**
  * Temporary workaround to patch selenium's element handle API with methods
  * that match the playwright API for Elements
@@ -1097,7 +1105,7 @@ class Driver {
     }
 
     throw new Error(
-      `waitUntilXWindowHandles timed out polling window handles. Expected: ${x}, Actual: ${windowHandles.length}`,
+      `${errorMessages.waitUntilXWindowHandlesTimeout}. Expected: ${x}, Actual: ${windowHandles.length}`,
     );
   }
 
@@ -1569,4 +1577,4 @@ function sanitizeTestTitle(testTitle) {
     .replace(/^-+|-+$/gu, ''); // Trim leading/trailing dashes
 }
 
-module.exports = { Driver, PAGES };
+module.exports = { Driver, PAGES, errorMessages };

types/global.d.ts

@@ -260,6 +260,11 @@ type StateHooks = {
    * the `onInstalled` listener is called.
    */
   metamaskTriggerOnInstall?: () => void;
+  /**
+   * Reload the extension. This is used to trigger extension reload from a page context by E2E
+   * tests.
+   */
+  reloadExtension?: () => void;
 };
 
 export declare global {

ui/index.js

@@ -294,6 +294,15 @@ function setupStateHooks(store) {
     };
   }
 
+  /**
+   * Reload the extension.
+   *
+   * This is used for the `first-install` E2E test, which uses a production-like build. This
+   * function must be present even if `process.env.IN_TEST` is false.
+   */
+  window.stateHooks.reloadExtension = () => {
+    browser.runtime.reload();
+  };
   window.stateHooks.getCleanAppState = async function () {
     const state = clone(store.getState());
     // we use the manifest.json version from getVersion and not