E2e reliability and performance (#2201)

Author: samuelreichert

.github/workflows/RunE2ENightly.yml

@@ -2,56 +2,66 @@ name: Run E2E test nightly
 # This workflow is used to test our widgets nightly.
 
 on:
-  schedule:
-    # At 02:00 on every day-of-week.
-    - cron: "0 02 * * 1-5"
+    schedule:
+        # At 02:00 on every day-of-week.
+        - cron: "0 02 * * 1-5"
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
 
 jobs:
-  e2e:
-    name: Run automated end-to-end tests nightly
-    runs-on: ubuntu-latest
-
-    permissions:
-      packages: read
-      contents: read
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 0
-      - name: Setup pnpm
-        uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0
-
-      - name: Setup node
-        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
-        with:
-          node-version-file: ".nvmrc"
-          cache: "pnpm"
-
-      - name: Install dependencies
-        run: pnpm install
-
-      - name: Install Playwright Browsers
-        run: pnpm exec playwright install --with-deps chromium
-
-      - name: Executing E2E tests
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: pnpm -r --workspace-concurrency=1 --no-bail run e2e
-
-      - name: Fixing files permissions
-        if: failure()
-        run: |
-          sudo find ${{ github.workspace }}/packages/* -type d -exec chmod 755 {} \;
-          sudo find ${{ github.workspace }}/packages/* -type f -exec chmod 644 {} \;
-
-      - name: Archive test screenshot diff results
-        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        if: failure()
-        with:
-          name: test-screenshot-results
-          path: |
-            ${{ github.workspace }}/packages/**/**/test-results/**/*.png
-            ${{ github.workspace }}/packages/**/**/test-results/**/*.zip
-          if-no-files-found: error
+    e2e:
+        name: Run automated end-to-end tests nightly
+        runs-on: ubuntu-latest
+
+        permissions:
+            packages: read
+            contents: read
+
+        strategy:
+            fail-fast: false
+            matrix:
+                index: [0, 1, 2, 3]
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+              with:
+                  fetch-depth: 0
+            - name: Setup pnpm
+              uses: pnpm/action-setup@a3252b78c470c02df07e9d59298aecedc3ccdd6d # v3.0.0
+
+            - name: Setup node
+              uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
+              with:
+                  node-version-file: ".nvmrc"
+                  cache: "pnpm"
+
+            - name: Install dependencies
+              run: pnpm install
+
+            - name: Install Playwright Browsers
+              run: pnpm exec playwright install --with-deps chromium
+
+            - name: Executing E2E tests
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: >-
+                  node ./automation/run-e2e/bin/run-e2e-in-chunks.mjs --chunks 4 --index ${{ matrix.index }} --event-name ${{ github.event_name }}
+
+            - name: Fixing files permissions
+              if: failure()
+              run: |
+                  sudo find ${{ github.workspace }}/packages/* -type d -exec chmod 755 {} \;
+                  sudo find ${{ github.workspace }}/packages/* -type f -exec chmod 644 {} \;
+
+            - name: Archive test screenshot diff results
+              uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+              if: failure()
+              with:
+                  name: test-screenshot-results-${{ matrix.index }}
+                  path: |
+                      ${{ github.workspace }}/packages/**/**/test-results/**/*.png
+                      ${{ github.workspace }}/packages/**/**/test-results/**/*.zip
+                  if-no-files-found: error

AGENTS.md

@@ -33,6 +33,10 @@ docs/requirements/                -> Detailed technical requirements
 - Jest + RTL for unit tests (src/\*_/**tests**/_.spec.ts)
 - Playwright for E2E (e2e/\*.spec.js)
 
+## E2E Test Rules (Playwright)
+
+- docs/requirements/e2e-test-guidelines.md — Full rules + template
+
 ## Development Setup
 
 - Node >=22, pnpm 10.x
@@ -49,6 +53,7 @@ docs/requirements/                -> Detailed technical requirements
 ## Documentation
 
 Essential reading (consult for any widget work):
+
 - docs/repo-layout.md — To understand the repository
 - docs/requirements/tech-stack.md — Full technology stack
 - docs/requirements/frontend-guidelines.md — CSS/SCSS/Atlas UI guidelines
@@ -57,8 +62,10 @@ Essential reading (consult for any widget work):
 - docs/requirements/project-requirements-document.md — Goals and scope
 
 Reference (consult on demand for specific tasks):
+
 - docs/requirements/implementation-plan.md — New widget guide + PR template
 - docs/requirements/widget-to-module.md — Widget-to-module conversion guide
+- docs/requirements/e2e-test-guidelines.md — E2E test reliability rules + template
 
 ## Agent-Specific Instructions
 

automation/run-e2e/eslint.config.mjs

@@ -1,6 +1,7 @@
 import { defineConfig } from "eslint/config";
 import globals from "globals";
 import js from "@eslint/js";
+import playwright from "eslint-plugin-playwright";
 
 export default defineConfig([
     {
@@ -21,5 +22,14 @@ export default defineConfig([
         rules: {
             "no-unused-vars": "warn"
         }
+    },
+    {
+        files: ["**/e2e/**/*.spec.{,m,c}js"],
+        plugins: { playwright },
+        rules: {
+            "playwright/no-wait-for-timeout": "error",
+            "playwright/no-networkidle": "warn",
+            "playwright/prefer-web-first-assertions": "warn"
+        }
     }
 ]);

automation/run-e2e/lib/fixtures.mjs

@@ -0,0 +1,38 @@
+/* eslint-disable no-undef */
+import { test as base, expect } from "@playwright/test";
+
+async function waitForMendixApp(page, timeout = 60_000) {
+    await page.waitForLoadState("domcontentloaded");
+    await page.waitForFunction(
+        () =>
+            Boolean(window.mx?.session) &&
+            !document.querySelector(".mx-progress-indicator") &&
+            document.querySelector(".mx-page") !== null,
+        undefined,
+        { timeout }
+    );
+}
+
+export { expect };
+
+export const test = base.extend({
+    mendixSession: [
+        async ({ browser }, use) => {
+            const context = await browser.newContext();
+            const page = await context.newPage();
+            const originalGoto = page.goto.bind(page);
+            page.goto = async (url, options) => {
+                const response = await originalGoto(url, options);
+                await waitForMendixApp(page);
+                return response;
+            };
+            await use({ context, page });
+            await page.evaluate(() => window.mx?.session?.logout?.()).catch(() => {});
+            await context.close();
+        },
+        { scope: "worker" }
+    ],
+    page: async ({ mendixSession }, use) => {
+        await use(mendixSession.page);
+    }
+});

automation/run-e2e/lib/mendix-helpers.mjs

@@ -0,0 +1,57 @@
+/* eslint-disable no-undef */
+import { expect } from "@playwright/test";
+
+export async function waitForMendixApp(page, timeout = 60_000) {
+    await page.waitForLoadState("domcontentloaded");
+    await page.waitForFunction(
+        () =>
+            Boolean(window.mx?.session) &&
+            !document.querySelector(".mx-progress-indicator") &&
+            document.querySelector(".mx-page") !== null,
+        undefined,
+        { timeout }
+    );
+}
+
+export async function waitForDataReady(page, timeout = 60_000) {
+    await waitForMendixApp(page, timeout);
+    await page.waitForLoadState("networkidle");
+}
+
+export async function waitForWidget(page, mxName, timeout = 15_000) {
+    const locator = page.locator(`.mx-name-${mxName}`);
+    await expect(locator).toBeVisible({ timeout });
+    return locator;
+}
+
+export async function waitForListData(page, mxName, minRows = 1, timeout = 15_000) {
+    const container = page.locator(`.mx-name-${mxName}`);
+    await expect(container).toBeVisible({ timeout });
+    const rows = container.locator("[class*='item'], tr[class*='row'], [class*='gallery-item']");
+    await expect(rows).toHaveCount(minRows, { timeout });
+    return rows;
+}
+
+export async function safeLogout(page) {
+    await page.evaluate(() => window.mx?.session?.logout?.()).catch(() => {});
+}
+
+export async function navigateToPage(page, path, timeout = 30_000) {
+    await page.goto(path);
+    await waitForMendixApp(page, timeout);
+}
+
+export async function checkAccessibility(page, selector, options = {}) {
+    const AxeBuilder = (await import("@axe-core/playwright")).default;
+    let builder = new AxeBuilder({ page }).withTags(options.tags || ["wcag21aa"]);
+    if (selector) {
+        builder = builder.include(selector);
+    }
+    if (options.exclude) {
+        for (const sel of [].concat(options.exclude)) {
+            builder = builder.exclude(sel);
+        }
+    }
+    const results = await builder.analyze();
+    expect(results.violations).toEqual([]);
+}

automation/run-e2e/package.json

@@ -12,6 +12,11 @@
         "run-e2e": "bin/run-e2e.mjs"
     },
     "type": "module",
+    "exports": {
+        "./fixtures": "./lib/fixtures.mjs",
+        "./mendix-helpers": "./lib/mendix-helpers.mjs",
+        "./playwright.config.cjs": "./playwright.config.cjs"
+    },
     "scripts": {
         "format": "prettier --ignore-path ./node_modules/@mendix/prettier-config-web-widgets/global-prettierignore --write .",
         "lint": "eslint .",

automation/run-e2e/playwright.config.cjs

@@ -9,11 +9,13 @@ module.exports = defineConfig({
     fullyParallel: true,
     /* Fail the build on CI if you accidentally left test.only in the source code. */
     forbidOnly: !!process.env.CI,
+    /* Filter tests by tag: E2E_SUITE=smoke runs only @smoke-tagged tests */
+    grep: process.env.E2E_SUITE === "smoke" ? /@smoke/ : undefined,
     /* Retry on CI only */
     retries: process.env.CI ? 2 : 0,
-    /* Use 4 workers on CI – the runner has multiple cores and each widget's tests
-     * are independent, so parallel execution cuts per-widget runtime significantly. */
-    workers: process.env.CI ? 4 : undefined,
+    /* Worker-scoped session: each worker holds 1 Mendix session. Safe up to 4 workers
+     * against the 5-session developer license (leaves 1 headroom). */
+    workers: process.env.CI ? 4 : 2,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
     reporter: [
         ["list"],
@@ -35,11 +37,20 @@ module.exports = defineConfig({
             reuseExistingServer: !process.env.CI
         }
     ], */
+    expect: {
+        toHaveScreenshot: {
+            animations: "disabled",
+            threshold: 0.1
+        }
+    },
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
         /* Base URL to use in actions like `await page.goto('/')`. */
         baseURL: process.env.URL ? process.env.URL : "http://127.0.0.1:8080",
 
+        actionTimeout: 10_000,
+        navigationTimeout: 30_000,
+
         /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
         trace: "on-first-retry",