Skip to content

test: Add E2E test for first install #31435

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 31 additions & 5 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -174,14 +174,22 @@ workflows:
- test-e2e-swap-playwright - OPTIONAL:
requires:
- prep-build
- test-e2e-chrome-vault-decryption:
- test-e2e-chrome-dist:
filters:
branches:
only:
- main
- /^Version-v(\d+)[.](\d+)[.](\d+)/
requires:
- prep-build
- test-e2e-firefox-dist:
filters:
branches:
only:
- main
- /^Version-v(\d+)[.](\d+)[.](\d+)/
requires:
- prep-build-mv2
- validate-source-maps:
requires:
- prep-build
Expand Down Expand Up @@ -213,7 +221,8 @@ workflows:
- test-e2e-firefox
- test-e2e-chrome-flask
- test-e2e-firefox-flask
- test-e2e-chrome-vault-decryption
- test-e2e-firefox-dist
- test-e2e-chrome-dist
- test-e2e-chrome-webpack
- job-publish-prerelease:
requires:
Expand Down Expand Up @@ -776,16 +785,33 @@ jobs:
- store_test_results:
path: test/test-results/e2e

test-e2e-chrome-vault-decryption:
test-e2e-chrome-dist:
executor: node-browsers-medium-plus
steps:
- run: *shallow-git-clone-and-enable-vnc
- run: sudo corepack enable
- attach_workspace:
at: .
- run:
name: Run E2E tests that require a production-like build
command: .circleci/scripts/test-run-e2e.sh yarn test:e2e:chrome:dist
no_output_timeout: 5m
- store_artifacts:
path: test-artifacts
destination: test-artifacts
- store_test_results:
path: test/test-results/e2e

test-e2e-firefox-dist:
executor: node-browsers-medium-plus
steps:
- run: *shallow-git-clone-and-enable-vnc
- run: sudo corepack enable
- attach_workspace:
at: .
- run:
name: test:e2e:single
command: .circleci/scripts/test-run-e2e.sh yarn test:e2e:single test/e2e/vault-decryption-chrome.spec.ts --browser chrome
name: Run E2E tests that require a production-like build
command: .circleci/scripts/test-run-e2e.sh yarn test:e2e:firefox:dist
no_output_timeout: 5m
- store_artifacts:
path: test-artifacts
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/e2e-chrome.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,12 @@ jobs:
matrix-index: ${{ matrix.index }}
matrix-total: ${{ strategy.job-total }}

test-e2e-chrome-vault-decryption:
test-e2e-chrome-dist:
uses: ./.github/workflows/run-e2e.yml
with:
test-suite-name: test-e2e-chrome-vault-decryption
test-suite-name: test-e2e-chrome-dist
build-artifact: prep-build-dist-browserify-chrome
test-command: yarn test:e2e:single test/e2e/vault-decryption-chrome.spec.ts --browser chrome
test-command: yarn test:e2e:chrome:dist

test-e2e-chrome-report:
needs:
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/e2e-firefox.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@ jobs:
matrix-index: ${{ matrix.index }}
matrix-total: ${{ strategy.job-total }}

test-e2e-firefox-dist:
uses: ./.github/workflows/run-e2e.yml
with:
test-suite-name: test-e2e-firefox-dist
build-artifact: prep-build-dist-browserify-firefox
test-command: yarn test:e2e:firefox:dist

test-e2e-firefox-report:
needs:
- test-e2e-firefox
Expand Down
50 changes: 43 additions & 7 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ jobs:
- prep-deps
uses: ./.github/workflows/validate-lavamoat-policy-webapp.yml

prep-build-test-browserify:
prep-build-test-browserify-chrome:
needs:
- prep-deps
runs-on: ubuntu-latest
Expand All @@ -97,6 +97,19 @@ jobs:
name: prep-build-test-browserify-chrome
path: ./dist/**/chrome # it's really in dist/chrome, but the ** is there so that it's uploaded relative to `dist`

prep-build-test-browserify-firefox:
needs:
- prep-deps
runs-on: ubuntu-latest
steps:
- name: Checkout and setup environment
uses: MetaMask/action-checkout-and-setup@v1
with:
is-high-risk-environment: false
skip-allow-scripts: true

- run: yarn build:test:mv2

- name: Upload artifact prep-build-test-browserify-firefox
uses: actions/upload-artifact@v4
with:
Expand Down Expand Up @@ -128,7 +141,7 @@ jobs:
name: prep-build-test-webpack-firefox
path: ./dist/**/firefox # it's really in dist/firefox, but the ** is there so that it's uploaded relative to `dist`

prep-build-dist-browserify:
prep-build-dist-browserify-chrome:
needs:
- prep-deps
runs-on: ubuntu-latest
Expand All @@ -150,6 +163,26 @@ jobs:
name: prep-build-dist-browserify-chrome
path: ./dist/**/chrome # it's really in dist/chrome, but the ** is there so that it's uploaded relative to `dist`

prep-build-dist-browserify-firefox:
needs:
- prep-deps
runs-on: ubuntu-latest
env:
# For a `pull_request` event, the branch is `github.head_ref``.
# For a `push` event, the branch is `github.ref_name`.
BRANCH: ${{ github.head_ref || github.ref_name }}
steps:
- name: Checkout and setup environment
uses: MetaMask/action-checkout-and-setup@v1
with:
is-high-risk-environment: false
skip-allow-scripts: true

# GHA does not allow the : ? ternary operator, you must write && ||
- run: ${{ env.BRANCH == 'master' && 'ENABLE_MV3=false yarn build prod' || 'yarn dist:mv2' }}
env:
INFURA_PROJECT_ID: 'FAKE'

- name: Upload artifact prep-build-dist-browserify-firefox
uses: actions/upload-artifact@v4
with:
Expand All @@ -159,7 +192,7 @@ jobs:
run-benchmarks:
uses: ./.github/workflows/run-benchmarks.yml
needs:
- prep-build-test-browserify
- prep-build-test-browserify-chrome
- prep-build-test-webpack
permissions:
contents: read
Expand All @@ -174,7 +207,7 @@ jobs:

bundle-size:
needs:
- prep-build-dist-browserify
- prep-build-dist-browserify-chrome
runs-on: ubuntu-latest
env:
EXTENSION_BUNDLESIZE_STATS_TOKEN: ${{ secrets.EXTENSION_BUNDLESIZE_STATS_TOKEN }}
Expand Down Expand Up @@ -221,15 +254,17 @@ jobs:
e2e-chrome:
needs:
- needs-e2e
- prep-build-test-browserify
- prep-build-test-browserify-chrome
- prep-build-test-webpack
- prep-build-dist-browserify
- prep-build-dist-browserify-chrome
if: ${{ needs.needs-e2e.outputs.needs-e2e == 'true' }}
uses: ./.github/workflows/e2e-chrome.yml

e2e-firefox:
needs:
- needs-e2e
- prep-build-test-browserify-firefox
- prep-build-dist-browserify-firefox
if: ${{ needs.needs-e2e.outputs.needs-e2e == 'true' }}
uses: ./.github/workflows/e2e-firefox.yml

Expand Down Expand Up @@ -292,7 +327,8 @@ jobs:
- test-storybook
- validate-lavamoat-policy-build
- validate-lavamoat-policy-webapp
- prep-build-test-browserify
- prep-build-test-browserify-chrome
- prep-build-test-browserify-firefox
- prep-build-test-webpack
- run-tests
- needs-e2e
Expand Down
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ app/.DS_Store
storybook-build/
coverage/
jest-coverage/
dist
/dist
builds*/
builds.zip
development/ts-migration-dashboard/build
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
"test:integration": "npx webpack build --config ./development/webpack/webpack.integration.tests.config.ts && jest --config jest.integration.config.js",
"test:integration:coverage": "yarn test:integration --coverage",
"test:e2e:chrome": "SELENIUM_BROWSER=chrome node test/e2e/run-all.js",
"test:e2e:chrome:dist": "SELENIUM_BROWSER=chrome node test/e2e/run-all.js --dist",
"test:e2e:chrome:flask": "SELENIUM_BROWSER=chrome node test/e2e/run-all.js --build-type flask",
"test:e2e:chrome:webpack": "SELENIUM_BROWSER=chrome node test/e2e/run-all.js",
"test:api-specs": "SELENIUM_BROWSER=chrome ts-node test/e2e/run-openrpc-api-test-coverage.ts",
Expand All @@ -63,6 +64,7 @@
"test:e2e:chrome:rpc": "SELENIUM_BROWSER=chrome node test/e2e/run-all.js --rpc",
"test:e2e:chrome:multi-provider": "MULTIPROVIDER=true SELENIUM_BROWSER=chrome node test/e2e/run-all.js --multi-provider",
"test:e2e:firefox": "SELENIUM_BROWSER=firefox node test/e2e/run-all.js",
"test:e2e:firefox:dist": "SELENIUM_BROWSER=firefox node test/e2e/run-all.js --dist",
"test:e2e:firefox:flask": "SELENIUM_BROWSER=firefox node test/e2e/run-all.js --build-type flask",
"test:e2e:single": "node test/e2e/run-e2e-test.js",
"ganache:start": "./development/run-ganache.sh",
Expand Down
53 changes: 53 additions & 0 deletions test/e2e/dist/first-install.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import assert from 'assert/strict';
import { Browser } from 'selenium-webdriver';
import { withFixtures } from '../helpers';
import { errorMessages } from '../webdriver/driver';
import StartOnboardingPage from '../page-objects/pages/onboarding/start-onboarding-page';
import { hasProperty, isObject } from '@metamask/utils';

describe('First install', function () {
it('opens new window upon install, but not on subsequent reloads', async function () {
await withFixtures(
{
disableServerMochaToBackground: true,
},
async ({ driver }) => {
// Wait for MetaMask to automatically open a new tab
await driver.waitUntilXWindowHandles(2);

let windowHandles = await driver.getAllWindowHandles();

// Switch to new tab and verify it's the start onboarding page
await driver.driver.switchTo().window(windowHandles[1]);
const startOnboardingPage = new StartOnboardingPage(driver);
await startOnboardingPage.check_pageIsLoaded();

await driver.executeScript('window.stateHooks.reloadExtension()');

// Wait for extension to reload, signified by the onboarding tab closing
await driver.waitUntilXWindowHandles(1);

// Test to see if it re-opens
try {
await driver.waitUntilXWindowHandles(2);
} catch (error) {
if (
isObject(error) &&
hasProperty(error, 'message') &&
typeof error.message === 'string' &&
error.message.startsWith(
errorMessages.waitUntilXWindowHandlesTimeout,
)
) {
// Ignore timeout error, it's expected here in the success case
console.log('Onboarding tab not opened');
return;
} else {
throw error;
}
}
throw new Error('Onboarding tab opened unexpectedly');
},
);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@ import os from 'os';
import path from 'path';
import fs from 'fs-extra';
import level from 'level';
import { Driver } from './webdriver/driver';
import { WALLET_PASSWORD, WINDOW_TITLES, withFixtures } from './helpers';
import HeaderNavbar from './page-objects/pages/header-navbar';
import HomePage from './page-objects/pages/home/homepage';
import PrivacySettings from './page-objects/pages/settings/privacy-settings';
import SettingsPage from './page-objects/pages/settings/settings-page';
import VaultDecryptorPage from './page-objects/pages/vault-decryptor-page';
import { completeCreateNewWalletOnboardingFlowWithCustomSettings } from './page-objects/flows/onboarding.flow';
import { Driver } from '../webdriver/driver';
import { WALLET_PASSWORD, WINDOW_TITLES, withFixtures } from '../helpers';
import HeaderNavbar from '../page-objects/pages/header-navbar';
import HomePage from '../page-objects/pages/home/homepage';
import PrivacySettings from '../page-objects/pages/settings/privacy-settings';
import SettingsPage from '../page-objects/pages/settings/settings-page';
import VaultDecryptorPage from '../page-objects/pages/vault-decryptor-page';
import { completeCreateNewWalletOnboardingFlowWithCustomSettings } from '../page-objects/flows/onboarding.flow';

const VAULT_DECRYPTOR_PAGE = 'https://metamask.github.io/vault-decryptor';

Expand Down Expand Up @@ -157,6 +157,10 @@ async function closePopoverIfPresent(driver: Driver) {

describe('Vault Decryptor Page', function () {
it('is able to decrypt the vault uploading the log file in the vault-decryptor webapp', async function () {
if (process.env.SELENIUM_BROWSER !== 'chrome') {
// TODO: Get this working on Firefox
this.skip();
}
await withFixtures(
{
disableServerMochaToBackground: true,
Expand Down Expand Up @@ -215,6 +219,10 @@ describe('Vault Decryptor Page', function () {
});

it('is able to decrypt the vault pasting the text in the vault-decryptor webapp', async function () {
if (process.env.SELENIUM_BROWSER !== 'chrome') {
// TODO: Get this working on Firefox
this.skip();
}
await withFixtures(
{
disableServerMochaToBackground: true,
Expand Down
8 changes: 8 additions & 0 deletions test/e2e/run-all.js
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,10 @@ async function main() {
'Run tests in debug mode, logging each driver interaction',
type: 'boolean',
})
.option('dist', {
description: `run e2e tests for production-like builds`,
type: 'boolean',
})
.option('rpc', {
description: `run json-rpc specific e2e tests`,
type: 'boolean',
Expand Down Expand Up @@ -219,6 +223,7 @@ async function main() {
const {
browser,
debug,
dist,
retries,
rpc,
buildType,
Expand All @@ -243,6 +248,9 @@ async function main() {
...(await getTestPathsForTestDir(path.join(__dirname, 'flask'))),
...featureTestsOnMain,
];
} else if (dist) {
const testDir = path.join(__dirname, 'dist');
testPaths = await getTestPathsForTestDir(testDir);
} else if (rpc) {
const testDir = path.join(__dirname, 'json-rpc');
testPaths = await getTestPathsForTestDir(testDir);
Expand Down
Loading
Loading