Added Playwright test for bruno-testbench, few sanity tests and improvements

- Trace will capture snapshots now
- Added ability to add init Electron user-data, preferences and other
  app settings.
- Improved test Fixtures
  - Use tempdir for Electron user-data
  - Ability to reuse app instance for a given init user-data by placing
    them in a folder(`pageWithUserData` Fixture)
  - Ability to create tests with fresh user-data(`newPage` Fixture)
- Improved logging
- Improved the env vars to customize the Electron user-data-path

Author: ramki-bruno

contributing.md

@@ -99,14 +99,13 @@ npm run dev
 ```
 
 #### Customize Electron `userData` path
-If `ELECTRON_APP_NAME` env-variable is present and its development mode, then the `appName` and `userData` path is modified accordingly.
+If `ELECTRON_USER_DATA_PATH` env-variable is present and its development mode, then `userData` path is modified accordingly.
 
 e.g.
 ```sh
-ELECTRON_APP_NAME=bruno-dev npm run dev:electron
+ELECTRON_USER_DATA_PATH=$(realpath ~/Desktop/bruno-test) npm run dev:electron
 ```
-
-> This doesn't change the name of the window or the names in lot of other places, only the name used by Electron internally.
+This will create a `bruno-test` folder on your Desktop and use it as the `userData` path.
 
 ### Troubleshooting
 

e2e-tests/001-sanity-tests/001-home-screen.spec.ts

@@ -0,0 +1,5 @@
+import { test, expect } from '../../playwright';
+
+test('Check if the logo on top left is visible', async ({ page }) => {
+  await expect(page.getByRole('button', { name: 'bruno' })).toBeVisible();
+});

e2e-tests/001-sanity-tests/002-create-new-collection.spec.ts

@@ -0,0 +1,31 @@
+import { test, expect } from '../../playwright';
+
+test('Create new collection and add a simple HTTP request', async ({ page, createTmpDir }) => {
+  await page.getByLabel('Create Collection').click();
+  await page.getByLabel('Name').click();
+  await page.getByLabel('Name').fill('test-collection');
+  await page.getByLabel('Name').press('Tab');
+  await page.getByLabel('Location').fill(await createTmpDir('test-collection'));
+  await page.getByRole('button', { name: 'Create', exact: true }).click();
+  await page.getByText('test-collection').click();
+  await page.getByLabel('Safe ModeBETA').check();
+  await page.getByRole('button', { name: 'Save' }).click();
+  await page.locator('#create-new-tab').getByRole('img').click();
+  await page.getByPlaceholder('Request Name').fill('r1');
+  await page.getByPlaceholder('Request URL').click();
+  await page.getByPlaceholder('Request URL').fill('http://localhost:8081');
+  await page.getByRole('button', { name: 'Create' }).click();
+  await page.locator('pre').filter({ hasText: 'http://localhost:' }).click();
+  await page.locator('textarea').fill('/ping');
+  await page.locator('#send-request').getByRole('img').nth(2).click();
+
+  await expect(page.getByRole('main')).toContainText('200 OK');
+
+  await page.getByRole('tab', { name: 'GET r1' }).locator('circle').click();
+  await page.getByRole('button', { name: 'Save', exact: true }).click();
+  await page.getByText('GETr1').click();
+  await page.getByRole('button', { name: 'Clear response' }).click();
+  await page.locator('body').press('ControlOrMeta+Enter');
+
+  await expect(page.getByRole('main')).toContainText('200 OK');
+});

e2e-tests/bruno-testbench/init-user-data/preferences.json

@@ -0,0 +1,4 @@
+{
+  "maximized": true,
+  "lastOpenedCollections": ["{{projectRoot}}/packages/bruno-tests/collection"]
+}

e2e-tests/bruno-testbench/run-testbench-requests.spec.ts

@@ -0,0 +1,49 @@
+import { test, expect } from '../../playwright';
+
+test.describe.parallel('Run Testbench Requests', () => {
+  test('Run bruno-testbench in Developer Mode', async ({ pageWithUserData: page }) => {
+    test.setTimeout(2 * 60 * 1000);
+
+    await page.getByText('bruno-testbench').click();
+    await page.getByLabel('Developer Mode(use only if').check();
+    await page.getByRole('button', { name: 'Save' }).click();
+    await page.locator('.environment-selector').nth(1).click();
+    await page.locator('.dropdown-item').getByText('Prod').click();
+    await page.locator('.collection-actions').hover();
+    await page.locator('.collection-actions .icon').click();
+    await page.getByText('Run', { exact: true }).click();
+    await page.getByRole('button', { name: 'Run Collection' }).click();
+    await page.getByRole('button', { name: 'Run Again' }).waitFor({ timeout: 2 * 60 * 1000 });
+
+    const result = await page.getByText('Total Requests: ').innerText();
+    const [totalRequests, passed, failed, skipped] = result
+      .match(/Total Requests: (\d+), Passed: (\d+), Failed: (\d+), Skipped: (\d+)/)
+      .slice(1);
+
+    await expect(parseInt(failed)).toBe(0);
+    await expect(parseInt(passed)).toBe(parseInt(totalRequests) - parseInt(skipped) - 1);
+  });
+
+  test.fixme('Run bruno-testbench in Safe Mode', async ({ pageWithUserData: page }) => {
+    test.setTimeout(2 * 60 * 1000);
+
+    await page.getByText('bruno-testbench').click();
+    await page.getByLabel('Safe ModeBETA').check();
+    await page.getByRole('button', { name: 'Save' }).click();
+    await page.locator('.environment-selector').nth(1).click();
+    await page.locator('.dropdown-item').getByText('Prod').click();
+    await page.locator('.collection-actions').hover();
+    await page.locator('.collection-actions .icon').click();
+    await page.getByText('Run', { exact: true }).click();
+    await page.getByRole('button', { name: 'Run Collection' }).click();
+    await page.getByRole('button', { name: 'Run Again' }).waitFor({ timeout: 2 * 60 * 1000 });
+
+    const result = await page.getByText('Total Requests: ').innerText();
+    const [totalRequests, passed, failed, skipped] = result
+      .match(/Total Requests: (\d+), Passed: (\d+), Failed: (\d+), Skipped: (\d+)/)
+      .slice(1);
+
+    await expect(parseInt(failed)).toBe(0);
+    await expect(parseInt(passed)).toBe(parseInt(totalRequests) - parseInt(skipped) - 1);
+  });
+});

e2e-tests/test-app-start.spec.ts

@@ -14,16 +14,11 @@ const { format } = require('url');
 const { BrowserWindow, app, session, Menu, ipcMain } = require('electron');
 const { setContentSecurityPolicy } = require('electron-util');
 
-if (isDev && process.env.ELECTRON_APP_NAME) {
-  const appName = process.env.ELECTRON_APP_NAME;
-  const userDataPath = path.join(app.getPath("appData"), appName);
+if (isDev && process.env.ELECTRON_USER_DATA_PATH) {
+  console.debug("`ELECTRON_USER_DATA_PATH` found, modifying `userData` path: \n"
+    + `\t${app.getPath("userData")} -> ${process.env.ELECTRON_USER_DATA_PATH}`);
 
-  console.log("`ELECTRON_APP_NAME` found, overriding `appName` and `userData` path: \n"
-    + `\t${app.getName()} -> ${appName}\n`
-    + `\t${app.getPath("userData")} -> ${userDataPath}`);
-
-  app.setName(appName);
-  app.setPath("userData", userDataPath);
+  app.setPath('userData', process.env.ELECTRON_USER_DATA_PATH);
 }
 
 const menuTemplate = require('./app/menu-template');

packages/bruno-electron/src/index.js

@@ -1,21 +1,21 @@
 import { defineConfig, devices } from '@playwright/test';
 
-const reporter: string[][string] = [['list'], ['html']];
+const reporter: any[] = [['list'], ['html']];
 
 if (process.env.CI) {
-  reporter.push(["github"]);
+  reporter.push(['github']);
 }
 
-
 export default defineConfig({
   testDir: './e2e-tests',
   fullyParallel: false,
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 1 : 0,
   workers: process.env.CI ? undefined : 1,
   reporter,
+
   use: {
-    trace: 'on-first-retry'
+    trace: process.env.CI ? 'on-first-retry' : 'on'
   },
 
   projects: [
@@ -24,9 +24,16 @@ export default defineConfig({
     }
   ],
 
-  webServer: {
-    command: 'npm run dev:web',
-    url: 'http://localhost:3000',
-    reuseExistingServer: !process.env.CI
-  }
+  webServer: [
+    {
+      command: 'npm run dev:web',
+      url: 'http://localhost:3000',
+      reuseExistingServer: !process.env.CI
+    },
+    {
+      command: 'npm start --workspace=packages/bruno-tests',
+      url: 'http://localhost:8081/ping',
+      reuseExistingServer: !process.env.CI
+    }
+  ]
 });

playwright.config.ts

@@ -7,7 +7,11 @@ exports.startApp = async () => {
   const app = await electron.launch({ args: [electronAppPath] });
   const context = await app.context();
 
-  app.process().stdout.on('data', (data) => console.log(data.toString()));
-  app.process().stderr.on('data', (error) => console.error(error.toString()));
+  app.process().stdout.on('data', (data) => {
+    process.stdout.write(data.toString().replace(/^(?=.)/gm, '[Electron] |'));
+  });
+  app.process().stderr.on('data', (error) => {
+    process.stderr.write(error.toString().replace(/^(?=.)/gm, '[Electron] |'));
+  });
   return { app, context };
 };

playwright/electron.ts

@@ -1,23 +1,179 @@
-import { test as baseTest, ElectronApplication, Page } from '@playwright/test';
+import { test as baseTest, BrowserContext, ElectronApplication, Page } from '@playwright/test';
+import * as path from 'path';
+import * as os from 'os';
+import * as fs from 'fs';
 
-const { startApp } = require('./electron.ts');
+const electronAppPath = path.join(__dirname, '../packages/bruno-electron');
 
-export const test = baseTest.extend<{ page: Page }, { electronApp: ElectronApplication }>({
-  electronApp: [
+export const test = baseTest.extend<
+  {
+    context: BrowserContext;
+    page: Page;
+    newPage: Page;
+    pageWithUserData: Page;
+  },
+  {
+    createTmpDir: (tag?: string) => Promise<string>;
+    launchElectronApp: (options?: { initUserDataPath?: string }) => Promise<ElectronApplication>;
+    electronApp: ElectronApplication;
+    reuseOrLaunchElectronApp: (options?: { initUserDataPath?: string }) => Promise<ElectronApplication>;
+  }
+>({
+  createTmpDir: [
     async ({}, use) => {
-      const { app: electronApp, context } = await startApp();
+      const dirs: string[] = [];
+      await use(async (tag?: string) => {
+        const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), `pw-${tag || ''}-`));
+        dirs.push(dir);
+        return dir;
+      });
+      await Promise.all(
+        dirs.map((dir) => fs.promises.rm(dir, { recursive: true, force: true, maxRetries: 10 }).catch((e) => e))
+      );
+    },
+    { scope: 'worker' }
+  ],
+
+  launchElectronApp: [
+    async ({ playwright, createTmpDir }, use, workerInfo) => {
+      const apps: ElectronApplication[] = [];
+      await use(async ({ initUserDataPath } = {}) => {
+        const userDataPath = await createTmpDir('electron-userdata');
+
+        if (initUserDataPath) {
+          const replacements = {
+            projectRoot: path.join(__dirname, '..')
+          };
 
-      await use(electronApp);
-      await context.close();
-      await electronApp.close();
+          for (const file of await fs.promises.readdir(initUserDataPath)) {
+            let content = await fs.promises.readFile(path.join(initUserDataPath, file), 'utf-8');
+            content = content.replace(/{{(\w+)}}/g, (_, key) => {
+              if (replacements[key]) {
+                return replacements[key];
+              } else {
+                throw new Error(`\tNo replacement for {{${key}}} in ${path.join(initUserDataPath, file)}`);
+              }
+            });
+            await fs.promises.writeFile(path.join(userDataPath, file), content, 'utf-8');
+          }
+        }
+
+        const app = await playwright._electron.launch({
+          args: [electronAppPath],
+          env: {
+            ...process.env,
+            ELECTRON_USER_DATA_PATH: userDataPath,
+          }
+        });
+
+        const { workerIndex } = workerInfo;
+        app.process().stdout.on('data', (data) => {
+          process.stdout.write(data.toString().replace(/^(?=.)/gm, `[Electron #${workerIndex}] |`));
+        });
+        app.process().stderr.on('data', (error) => {
+          process.stderr.write(error.toString().replace(/^(?=.)/gm, `[Electron #${workerIndex}] |`));
+        });
+
+        apps.push(app);
+        return app;
+      });
+      for (const app of apps) {
+        await app.context().close();
+        await app.close();
+      }
     },
     { scope: 'worker' }
   ],
-  page: async ({ electronApp }, use) => {
+
+  electronApp: [
+    async ({ launchElectronApp }, use) => {
+      const app = await launchElectronApp();
+      await use(app);
+    },
+    { scope: 'worker' }
+  ],
+
+  context: async ({ electronApp }, use, testInfo) => {
+    const context = await electronApp.context();
+    const tracingOptions = (testInfo as any)._tracing.traceOptions();
+    if (tracingOptions) {
+      try {
+        await context.tracing.start({ screenshots: true, snapshots: true, sources: true });
+      } catch (e) {}
+    }
+    await use(context);
+  },
+
+  page: async ({ electronApp, context }, use, testInfo) => {
     const page = await electronApp.firstWindow();
-    await use(page);
-    await page.reload();
+    const tracingOptions = (testInfo as any)._tracing.traceOptions();
+    if (tracingOptions) {
+      const tracePath = testInfo.outputPath(`trace-${testInfo.testId}.zip`);
+      await context.tracing.startChunk();
+      await use(page);
+      await context.tracing.stopChunk({ path: tracePath });
+      await testInfo.attach('trace', { path: tracePath });
+    } else {
+      await use(page);
+    }
+  },
+
+  newPage: async ({ launchElectronApp }, use, testInfo) => {
+    const app = await launchElectronApp();
+    const context = await app.context();
+    const page = await app.firstWindow();
+    const tracingOptions = (testInfo as any)._tracing.traceOptions();
+    if (tracingOptions) {
+      const tracePath = testInfo.outputPath(`trace-${testInfo.testId}.zip`);
+      await context.tracing.start({ screenshots: true, snapshots: true, sources: true });
+      await use(page);
+      await context.tracing.stop({ path: tracePath });
+      await testInfo.attach('trace', { path: tracePath });
+    } else {
+      await use(page);
+    }
+  },
+
+  reuseOrLaunchElectronApp: [
+    async ({ launchElectronApp }, use, testInfo) => {
+      const apps: Record<string, ElectronApplication> = {};
+      await use(async ({ initUserDataPath } = {}) => {
+        const key = initUserDataPath;
+        if (key && apps[key]) {
+          return apps[key];
+        }
+        const app = await launchElectronApp({ initUserDataPath });
+        apps[key] = app;
+        return app;
+      });
+    },
+    { scope: 'worker' }
+  ],
+
+  pageWithUserData: async ({ reuseOrLaunchElectronApp }, use, testInfo) => {
+    const testDir = path.dirname(testInfo.file);
+    const initUserDataPath = path.join(testDir, 'init-user-data');
+
+    const app = await reuseOrLaunchElectronApp(
+      (await fs.promises.stat(initUserDataPath).catch(() => false)) ? { initUserDataPath } : {}
+    );
+
+    const context = await app.context();
+    const page = await app.firstWindow();
+    const tracingOptions = (testInfo as any)._tracing.traceOptions();
+    if (tracingOptions) {
+      const tracePath = testInfo.outputPath(`trace-${testInfo.testId}.zip`);
+      try {
+        await context.tracing.start({ screenshots: true, snapshots: true, sources: true });
+      } catch (e) {}
+      await context.tracing.startChunk();
+      await use(page);
+      await context.tracing.stopChunk({ path: tracePath });
+      await testInfo.attach('trace', { path: tracePath });
+    } else {
+      await use(page);
+    }
   }
 });
 
-export * from '@playwright/test'
+export * from '@playwright/test';