ci: move all e2e vscode tests to run in ci

Author: benfdking

.github/workflows/pr.yaml

@@ -10,9 +10,9 @@ concurrency:
   cancel-in-progress: true
 jobs:
   test-vscode:
+    runs-on: ubuntu-latest
     env:
       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -26,7 +26,9 @@ jobs:
       - name: Run CI
         run: pnpm run ci
   test-vscode-e2e:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: ubuntu-runners
+      labels: ubuntu-2404-8
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -41,22 +43,43 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
+      - name: Cache pip dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key:
+            ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt',
+            '**/pyproject.toml', '**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Cache virtual environment
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key:
+            ${{ runner.os }}-venv-${{ hashFiles('**/requirements*.txt',
+            '**/pyproject.toml', '**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-venv-
       - name: Install python dependencies
         run: |
           python -m venv .venv
           source .venv/bin/activate
           make install-dev
-      - name: Fetch VS Code
-        working-directory: ./vscode/extension
-        run: pnpm run fetch-vscode
-
       - name: Install code-server
         run: curl -fsSL https://code-server.dev/install.sh | sh
       - name: Install Playwright browsers
         working-directory: ./vscode/extension
         run: pnpm exec playwright install
       - name: Run e2e tests
         working-directory: ./vscode/extension
+        timeout-minutes: 90
         run: |
           source ../../.venv/bin/activate
-          pnpm run test:e2e tests/stop.spec.ts
+          pnpm run test:e2e
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: vscode/extension/playwright-report/
+          retention-days: 30

.prettierignore

@@ -20,6 +20,9 @@ vscode/extension/out
 vscode/extension/src_react
 vscode/extension/tsconfig.tsbuildinfo
 vscode/extension/.vscode-test/
+vscode/extension/playwright-report/
+vscode/extension/test-results/
+vscode/extension/.test_setup
 
 sqlmesh
 docs

vscode/extension/.gitignore

@@ -2,7 +2,9 @@ node_modules
 dist
 out
 .vscode-test
+.test_setup
 *.vsix
 LICENSE
 src_react
-!src_react/.gitkeep
+!src_react/.gitkeep
+playwright-report

vscode/extension/.vscodeignore

@@ -24,4 +24,6 @@ tsconfig.test.json
 tsconfig.build.json
 src/test/**
 tests/**
-.claude
+.claude
+.idea
+.test_setup

vscode/extension/E2E_TESTING.md

@@ -9,16 +9,9 @@ This directory contains end-to-end tests for the SQLMesh VS Code extension using
    pnpm install
    ```
 
-2. **Download VS Code executable (one-time setup):**
+2. **Install Playwright browsers:**
    ```bash
-   pnpm run fetch-vscode
-   ```
-   
-   This downloads VS Code and caches it in `.vscode-test/` directory. The paths are saved to `.vscode-test/paths.json` for Playwright to use.
-
-3. **Install Playwright browsers:**
-   ```bash
-   npx playwright install
+   pnpm run playwright:install
    ```
 
 ## Running Tests

vscode/extension/package.json

@@ -124,7 +124,6 @@
     "test:e2e": "pnpm run vscode:package && playwright test",
     "test:e2e:ui": "pnpm run vscode:package && playwright test --ui",
     "test:e2e:headed": "pnpm run vscode:package && playwright test --headed",
-    "fetch-vscode": "tsx scripts/fetch-vscode.ts",
     "compile": "pnpm run check-types && node esbuild.js",
     "check-types": "tsc --noEmit -p ./tsconfig.build.json",
     "watch": "node esbuild.js --watch",

vscode/extension/playwright.config.ts

@@ -3,18 +3,22 @@ import { defineConfig } from '@playwright/test'
 export default defineConfig({
   testDir: 'tests',
   timeout: 60_000,
-  retries: process.env.CI ? 1 : 0,
-  workers: 1,
+  // TODO: When stable, allow retries in CI
+  retries: process.env.CI ? 0 : 0,
+  workers: process.env.CI ? 1 : 4,
+  reporter: [['html', { outputFolder: 'playwright-report' }], ['list']],
+  globalSetup: './tests/global-setup.ts',
   projects: [
     {
       name: 'electron-vscode',
       use: {
-        // ⭢ we'll launch Electron ourselves – no browser needed
         browserName: 'chromium',
-        headless: true, // headless mode for tests
+        headless: true,
         launchOptions: {
           slowMo: process.env.CI ? 0 : 100,
         },
+        viewport: { width: 1512, height: 944 },
+        video: 'retain-on-failure',
       },
     },
   ],

vscode/extension/tests/bad_setup.spec.ts

@@ -4,14 +4,17 @@ import os from 'os'
 import path from 'path'
 import {
   createVirtualEnvironment,
+  openFile,
   openLineageView,
   pipInstall,
   REPO_ROOT,
-  startVSCode,
   SUSHI_SOURCE_PATH,
 } from './utils'
+import { startCodeServer, stopCodeServer } from './utils_code_server'
 
-test('missing LSP dependencies shows install prompt', async ({}, testInfo) => {
+test('missing LSP dependencies shows install prompt', async ({
+  page,
+}, testInfo) => {
   testInfo.setTimeout(120_000) // 2 minutes for venv creation and package installation
   const tempDir = await fs.mkdtemp(
     path.join(os.tmpdir(), 'vscode-test-tcloud-'),
@@ -26,6 +29,11 @@ test('missing LSP dependencies shows install prompt', async ({}, testInfo) => {
   const sqlmeshWithExtras = `${REPO_ROOT}[bigquery]`
   await pipInstall(pythonDetails, [sqlmeshWithExtras, custom_materializations])
 
+  // Start VS Code
+  const context = await startCodeServer({
+    tempDir,
+  })
+
   try {
     // Copy sushi project
     await fs.copy(SUSHI_SOURCE_PATH, tempDir)
@@ -42,45 +50,45 @@ test('missing LSP dependencies shows install prompt', async ({}, testInfo) => {
       { spaces: 2 },
     )
 
-    // Start VS Code
-    const { window, close } = await startVSCode(tempDir)
+    await page.goto(`http://127.0.0.1:${context.codeServerPort}`)
 
     // Open a SQL file to trigger SQLMesh activation
     // Wait for the models folder to be visible
-    await window.waitForSelector('text=models')
+    await page.waitForSelector('text=models')
 
     // Click on the models folder
-    await window
+    await page
       .getByRole('treeitem', { name: 'models', exact: true })
       .locator('a')
       .click()
 
     // Open the top_waiters model
-    await window
+    await page
       .getByRole('treeitem', { name: 'customers.sql', exact: true })
       .locator('a')
       .click()
 
     // Wait for the message to show that LSP extras need to be installed
-    await window.waitForSelector('text=LSP dependencies missing')
-    expect(await window.locator('text=Install').count()).toBeGreaterThanOrEqual(
-      1,
-    )
-
-    await close()
+    await page.waitForSelector('text=LSP dependencies missing')
+    expect(await page.locator('text=Install').count()).toBeGreaterThanOrEqual(1)
   } finally {
-    // Clean up
-    await fs.remove(tempDir)
+    await stopCodeServer(context)
   }
 })
 
-test('lineage, no sqlmesh found', async ({}) => {
+test('lineage, no sqlmesh found', async ({ page }, testInfo) => {
+  testInfo.setTimeout(120_000) // 2 minutes for venv creation and package installation
+
   const tempDir = await fs.mkdtemp(
     path.join(os.tmpdir(), 'vscode-test-tcloud-'),
   )
   const pythonEnvDir = path.join(tempDir, '.venv')
   const pythonDetails = await createVirtualEnvironment(pythonEnvDir)
 
+  const context = await startCodeServer({
+    tempDir,
+  })
+
   try {
     // Copy sushi project
     await fs.copy(SUSHI_SOURCE_PATH, tempDir)
@@ -97,25 +105,28 @@ test('lineage, no sqlmesh found', async ({}) => {
       { spaces: 2 },
     )
 
-    const { window, close } = await startVSCode(tempDir)
+    // navigate to code-server instance
+    await page.goto(`http://127.0.0.1:${context.codeServerPort}`)
+    await page.waitForLoadState('networkidle')
 
     // Open lineage view
-    await openLineageView(window)
+    await openLineageView(page)
 
     // Assert shows that sqlmesh is not installed
-    await window.waitForSelector('text=SQLMesh LSP not found')
-
-    await close()
+    await page.waitForSelector('text=SQLMesh LSP not found')
   } finally {
     // Clean up
-    await fs.remove(tempDir)
+    await stopCodeServer(context)
   }
 })
 
 // Checks that if you have another file open like somewhere else, it still checks the workspace first for a successful context
 // it's very flaky but runs when debugging
 // - the typing in of the file name is very flaky
-test('check that the LSP runs correctly by opening lineage when looking at another file before not in workspace', async ({}) => {
+test('check that the LSP runs correctly by opening lineage when looking at another file before not in workspace', async ({
+  page,
+}, testInfo) => {
+  testInfo.setTimeout(120_000) // 2 minutes for venv creation and package installation
   const tempDir = await fs.mkdtemp(
     path.join(os.tmpdir(), 'vscode-test-tcloud-'),
   )
@@ -133,7 +144,7 @@ test('check that the LSP runs correctly by opening lineage when looking at anoth
   // Configure VS Code settings to use our Python environment
   const settings = {
     'python.defaultInterpreterPath': pythonDetails.pythonPath,
-    'sqlmesh.environmentPath': pythonEnvDir,
+    'sqlmesh.environmentPath': tempDir,
   }
   await fs.ensureDir(path.join(tempDir, '.vscode'))
   await fs.writeJson(path.join(tempDir, '.vscode', 'settings.json'), settings, {
@@ -148,19 +159,18 @@ test('check that the LSP runs correctly by opening lineage when looking at anoth
   await fs.ensureDir(path.dirname(sqlFile))
   await fs.writeFile(sqlFile, 'SELECT 1')
 
-  const { window, close } = await startVSCode(tempDir)
+  const context = await startCodeServer({
+    tempDir,
+  })
   try {
+    await page.goto(`http://127.0.0.1:${context.codeServerPort}`)
+    await page.waitForLoadState('networkidle')
+
     // Open the SQL file from the other directory
-    await window.keyboard.press('Meta+P')
-    await window.waitForTimeout(100)
-    await window.keyboard.type(sqlFile.toString(), { delay: 10 })
-    await window.waitForTimeout(100)
-    await window.keyboard.press('Enter')
-    await window.waitForTimeout(100)
-
-    await window.waitForSelector('text=Loaded SQLMesh context')
+    await openFile(page, sqlFile)
+
+    await page.waitForSelector('text=Loaded SQLMesh context')
   } finally {
-    await close()
-    await fs.remove(tempDir)
+    await stopCodeServer(context)
   }
 })