Migrate test running to the new vscode-test-cli (#4079)

vinistock · web-flow · commit 7d67236c688b · 2026-04-28T14:25:42.000Z
Migrate test running to the new vscode-test-cli (#4079) ### Motivation The test setup for VS Code extensions has changed a bit since the time when we created the Ruby LSP. This PR migrates us to the latest recommendations. ### Implementation Basically, we now use `@vscode/test-cli` which allows us to configure our tests in a declarative way. This allows us to get rid of some boiler plate files like `index.ts`. The other side of this is that we can now run tests through the test explorer panel, allowing us to easily run a single example at a time. Much better DX. Co-authored-by: vinistock <18742907+vinistock@users.noreply.github.com>
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -1,6 +1,7 @@
 {
   "recommendations": [
     "shopify.ruby-extensions-pack",
-    "dbaeumer.vscode-eslint"
+    "dbaeumer.vscode-eslint",
+    "ms-vscode.extension-test-runner"
   ]
 }
diff --git a/AGENTS.md b/AGENTS.md
@@ -198,5 +198,36 @@ is the client).
 
 ```bash
 pnpm run lint # Lint TypeScript code
-pnpm run test # Run extension tests
+pnpm run test # Run extension tests (compiles via the `pretest` hook, then runs vscode-test)
+```
+
+### Running targeted tests
+
+Pass flags through `pnpm run test --` so they reach `vscode-test` instead of being consumed by pnpm:
+
+```bash
+# Run a single compiled test file (use the .js path, not .ts)
+pnpm run test -- --run out/test/suite/workspace.test.js
+
+# Run several files at once
+pnpm run test -- --run out/test/suite/workspace.test.js --run out/test/suite/client.test.js
+
+# Filter by suite or test name (regex match against the full "suite > test" name)
+pnpm run test -- --grep "Workspace"
+pnpm run test -- --grep "does not restart when watched files"
+
+# Literal string match (no regex) — useful when the test name has regex metacharacters
+pnpm run test -- --fgrep "repeated rebase steps"
+
+# Invert the match — run everything except what `--grep`/`--fgrep` selects
+pnpm run test -- --grep "slow integration" --invert
+
+# Combine file scoping with name filtering for the fastest feedback loop
+pnpm run test -- --run out/test/suite/testController.test.js --grep "discovers tests"
+
+# List which tests would run without executing them (great for verifying a filter)
+pnpm run test -- --dry-run --grep "Workspace"
+
+# Stop at the first failure
+pnpm run test -- --bail --grep "Client"
 ```
diff --git a/vscode/.vscode-test.mjs b/vscode/.vscode-test.mjs
@@ -0,0 +1,9 @@
+import { defineConfig } from "@vscode/test-cli";
+
+export default defineConfig({
+  files: "out/test/suite/**/*.test.js",
+  mocha: {
+    ui: "tdd",
+    color: true,
+  },
+});
diff --git a/vscode/.vscode/launch.json b/vscode/.vscode/launch.json
@@ -2,6 +2,10 @@
 // Use IntelliSense to learn about possible attributes.
 // Hover to view descriptions of existing attributes.
 // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+//
+// To debug extension tests, install the "Extension Test Runner" extension
+// (ms-vscode.extension-test-runner). It reads .vscode-test.mjs and exposes
+// the tests in the Test Explorer with debugging support.
 {
 	"version": "0.2.0",
 	"configurations": [
@@ -16,19 +20,6 @@
 				"${workspaceFolder}/out/**/*.js"
 			],
 			"preLaunchTask": "${defaultBuildTask}"
-		},
-		{
-			"name": "Extension Tests",
-			"type": "extensionHost",
-			"request": "launch",
-			"args": [
-				"--extensionDevelopmentPath=${workspaceFolder}",
-				"--extensionTestsPath=${workspaceFolder}/out/test/suite/index"
-			],
-			"outFiles": [
-				"${workspaceFolder}/out/test/**/*.js"
-			],
-			"preLaunchTask": "${defaultBuildTask}"
 		}
 	]
 }
diff --git a/vscode/.vscode/settings.json b/vscode/.vscode/settings.json
@@ -0,0 +1,8 @@
+{
+  "files.exclude": {
+    "**/out": false
+  },
+  "search.exclude": {
+    "**/out": false
+  }
+}
diff --git a/vscode/.vscodeignore b/vscode/.vscodeignore
@@ -1,5 +1,6 @@
 .vscode/**
 .vscode-test/**
+.vscode-test.mjs
 src/**
 .gitignore
 vsc-extension-quickstart.md
diff --git a/vscode/eslint.config.mjs b/vscode/eslint.config.mjs
@@ -27,6 +27,7 @@ export default tseslint.config(
   {
     ignores: [
       "**/.vscode-test/**",
+      ".vscode-test.mjs",
       "eslint.config.mjs",
       "**/out/",
       "src/test/suite/fakeTestServer.js",
diff --git a/vscode/package.json b/vscode/package.json
@@ -810,7 +810,7 @@
     "pretest": "pnpm run compile",
     "format": "eslint . --fix && prettier '**/*.{ts,json,md,yaml,yml}' --write",
     "lint": "eslint . && prettier '**/*.{ts,json,md,yaml,yml}' --check",
-    "test": "node ./out/test/runTest.js"
+    "test": "vscode-test"
   },
   "pnpm": {
     "overrides": {
@@ -825,12 +825,12 @@
     "@types/node": "25.x",
     "@types/sinon": "^21.0.1",
     "@types/vscode": "^1.91.0",
+    "@vscode/test-cli": "^0.0.12",
     "@vscode/test-electron": "^2.5.2",
     "@vscode/vsce": "^3.9.1",
     "esbuild": "^0.28.0",
     "eslint": "^9.30.1",
     "eslint-plugin-prettier": "^5.5.5",
-    "glob": "^13.0.6",
     "mocha": "^11.7.5",
     "ovsx": "^0.10.11",
     "prettier": "^3.8.3",
diff --git a/vscode/pnpm-lock.yaml b/vscode/pnpm-lock.yaml
diff --git a/vscode/src/test/runTest.ts b/vscode/src/test/runTest.ts
diff --git a/vscode/src/test/suite/index.ts b/vscode/src/test/suite/index.ts

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"recommendations": [`
`3`	`3`	`"shopify.ruby-extensions-pack",`
`4`		`- "dbaeumer.vscode-eslint"`
	`4`	`+ "dbaeumer.vscode-eslint",`
	`5`	`+ "ms-vscode.extension-test-runner"`
`5`	`6`	`]`
`6`	`7`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,6 +27,7 @@ export default tseslint.config(`
`27`	`27`	`{`
`28`	`28`	`ignores: [`
`29`	`29`	`"/.vscode-test/",`
	`30`	`+ ".vscode-test.mjs",`
`30`	`31`	`"eslint.config.mjs",`
`31`	`32`	`"**/out/",`
`32`	`33`	`"src/test/suite/fakeTestServer.js",`