ci: stabilize integration tests with isolation, serial execution, chaos ordering, and comprehensive command coverage

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
     permissions:
       contents: read
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         node: [22, 24]
         camunda: ['8.8', '8.9']
@@ -69,7 +69,7 @@ jobs:
           echo "Camunda may not be fully ready, but continuing with tests..."
 
       - name: Run integration tests
-        run: npm run test:integration
+        run: npm run test:integration:chaos
         env:
           CAMUNDA_VERSION: ${{ matrix.camunda }}
 

package.json

@@ -29,7 +29,9 @@
     "prepublishOnly": "npm run build",
     "test": "node --test tests/unit/setup.test.ts tests/unit/*.test.ts tests/integration/*.test.ts",
     "test:unit": "node --test tests/unit/setup.test.ts tests/unit/*.test.ts",
-    "test:integration": "node --test tests/integration/*.test.ts",
+    "test:unit:chaos": "node tests/utils/chaos-runner.mjs tests/unit --setup-first tests/unit/setup.test.ts",
+    "test:integration": "node --test --test-concurrency=1 tests/integration/*.test.ts",
+    "test:integration:chaos": "node tests/utils/chaos-runner.mjs tests/integration",
     "dev": "node src/index.ts",
     "cli": "node src/index.ts"
   },

tests/integration/plugin-lifecycle.test.ts

@@ -7,57 +7,49 @@ import assert from 'node:assert';
 import { execSync, execFileSync } from 'node:child_process';
 import { existsSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
-import { getUserDataDir } from '../../src/config.ts';
+import { tmpdir } from 'node:os';
 
 describe('Plugin Lifecycle Integration Tests', () => {
   const testPluginDir = join(process.cwd(), 'test-plugin-temp');
   const testPluginName = 'c8ctl-test-plugin';
-  const pluginsDir = join(getUserDataDir(), 'plugins');
-  const nodeModulesPluginPath = join(pluginsDir, 'node_modules', testPluginName);
+  let testDataDir: string;
+  let pluginsDir: string;
+  let nodeModulesPluginPath: string;
+  let originalDataDir: string | undefined;
 
-  // Setup: Clean up any previous test artifacts
+  // Setup: Use an isolated data directory to avoid polluting the real user data dir
   before(() => {
+    originalDataDir = process.env.C8CTL_DATA_DIR;
+    testDataDir = join(tmpdir(), `c8ctl-plugin-test-${Date.now()}-${process.pid}`);
+    mkdirSync(testDataDir, { recursive: true });
+    process.env.C8CTL_DATA_DIR = testDataDir;
+
+    pluginsDir = join(testDataDir, 'plugins');
+    nodeModulesPluginPath = join(pluginsDir, 'node_modules', testPluginName);
+
     // Remove temp directory if it exists
     if (existsSync(testPluginDir)) {
       rmSync(testPluginDir, { recursive: true, force: true });
     }
-
-    // Unload plugin if it exists from previous run
-    try {
-      execSync(`node src/index.ts unload plugin ${testPluginName}`, { 
-        cwd: process.cwd(),
-        stdio: 'ignore' 
-      });
-    } catch {
-      // Ignore if not installed
-    }
-
-    // Remove from global node_modules if still there
-    if (existsSync(nodeModulesPluginPath)) {
-      rmSync(nodeModulesPluginPath, { recursive: true, force: true });
-    }
   });
 
-  // Cleanup: Ensure test artifacts are removed
+  // Cleanup: Restore environment and remove all test artifacts
   after(() => {
-    // Remove temp directory
+    // Restore original data dir
+    if (originalDataDir !== undefined) {
+      process.env.C8CTL_DATA_DIR = originalDataDir;
+    } else {
+      delete process.env.C8CTL_DATA_DIR;
+    }
+
+    // Remove temp plugin build directory
     if (existsSync(testPluginDir)) {
       rmSync(testPluginDir, { recursive: true, force: true });
     }
-
-    // Unload plugin
-    try {
-      execSync(`node src/index.ts unload plugin ${testPluginName}`, { 
-        cwd: process.cwd(),
-        stdio: 'ignore' 
-      });
-    } catch {
-      // Ignore if already uninstalled
-    }
-
-    // Remove from global node_modules
-    if (existsSync(nodeModulesPluginPath)) {
-      rmSync(nodeModulesPluginPath, { recursive: true, force: true });
+
+    // Remove isolated data directory (contains installed plugins)
+    if (existsSync(testDataDir)) {
+      rmSync(testDataDir, { recursive: true, force: true });
     }
   });
 

tests/integration/profile-switching.test.ts

@@ -6,10 +6,15 @@
 import { test, describe, beforeEach, afterEach } from 'node:test';
 import assert from 'node:assert';
 import { mkdirSync, rmSync, existsSync } from 'node:fs';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { tmpdir } from 'node:os';
+import { spawnSync } from 'node:child_process';
 import { pollUntil } from '../utils/polling.ts';
 
+const PROJECT_ROOT = resolve(import.meta.dirname, '..', '..');
+const CLI = join(PROJECT_ROOT, 'src', 'index.ts');
+const SPAWN_TIMEOUT_MS = 15_000;
+
 describe('Profile Switching Integration Tests', () => {
   let testDir: string;
   let originalEnv: NodeJS.ProcessEnv;
@@ -135,68 +140,34 @@ describe('Profile Switching Integration Tests', () => {
   });
 
   test('invalid profile causes connection error', async () => {
-    const { useProfile } = await import('../../src/commands/session.ts');
-    const { listProcessInstances } = await import('../../src/commands/process-instances.ts');
-    const { c8ctl } = await import('../../src/runtime.ts');
-
-    // Use the invalid profile
-    useProfile('invalid');
-    assert.strictEqual(c8ctl.activeProfile, 'invalid', 'Invalid profile should be active');
-
-    // Capture stderr to check for errors
-    const originalError = console.error;
-    const originalExit = process.exit;
-    let capturedErrors: string[] = [];
-    let exitCalled = false;
-    let exitCode: number | undefined;
-
-    console.error = (...args: any[]) => {
-      capturedErrors.push(args.join(' '));
-    };
-
-    process.exit = ((code?: number) => {
-      exitCalled = true;
-      exitCode = code;
-      throw new Error('process.exit called');
-    }) as any;
-
-    try {
-      // Attempt to list process instances - should fail
-      await listProcessInstances({
-        processDefinitionId: 'Process_0t60ay7',
+    // Use the CLI as a subprocess so that process.exit(1) happens in the child
+    // process and does not interfere with the test runner.
+    function cliWithProfile(...args: string[]) {
+      return spawnSync('node', [CLI, ...args], {
+        encoding: 'utf-8',
+        cwd: PROJECT_ROOT,
+        timeout: SPAWN_TIMEOUT_MS,
+        env: { ...process.env, C8CTL_DATA_DIR: testDir },
       });
-
-      assert.fail('Should have thrown an error or called process.exit');
-    } catch (error: any) {
-      // We expect either process.exit to be called or an error to be thrown
-      const errorOutput = capturedErrors.join('\n');
-
-      if (error.message === 'process.exit called') {
-        // process.exit was called, which is expected
-        assert.ok(exitCalled, 'process.exit should have been called');
-        assert.strictEqual(exitCode, 1, 'Exit code should be 1');
-        assert.ok(errorOutput.length > 0, 'Should have error output');
-        assert.ok(
-          errorOutput.includes('Failed to list process instances') ||
-          errorOutput.includes('ECONNREFUSED') ||
-          errorOutput.includes('connect') ||
-          errorOutput.includes('fetch failed'),
-          `Error should mention connection failure. Got: ${errorOutput}`
-        );
-      } else {
-        // An error was thrown, which is also acceptable
-        assert.ok(
-          error.message.includes('ECONNREFUSED') ||
-          error.message.includes('connect') ||
-          error.message.includes('Failed') ||
-          error.message.includes('fetch failed'),
-          `Error should mention connection failure. Got: ${error.message}`
-        );
-      }
-    } finally {
-      console.error = originalError;
-      process.exit = originalExit;
     }
+
+    // Switch to the invalid profile in the isolated data dir
+    const switchResult = cliWithProfile('use', 'profile', 'invalid');
+    assert.strictEqual(
+      switchResult.status, 0,
+      `'use profile invalid' should succeed. stderr: ${switchResult.stderr}`,
+    );
+
+    // Attempting to list process instances with the invalid profile should fail
+    const listResult = cliWithProfile('list', 'pi');
+    assert.strictEqual(listResult.status, 1, 'list pi with invalid profile should exit with code 1');
+    assert.ok(
+      listResult.stderr.includes('Failed to list process instances') ||
+      listResult.stderr.includes('ECONNREFUSED') ||
+      listResult.stderr.includes('connect') ||
+      listResult.stderr.includes('fetch failed'),
+      `stderr should mention connection failure. Got: ${listResult.stderr}`,
+    );
   });
 
   test('switching profiles affects cluster resolution', async () => {

tests/unit/completion.test.ts

@@ -4,22 +4,24 @@
 
 import { test, describe, beforeEach, afterEach } from 'node:test';
 import assert from 'node:assert';
+import { spawnSync } from 'node:child_process';
+import { join, resolve } from 'node:path';
 import { showCompletion } from '../../src/commands/completion.ts';
 
+const PROJECT_ROOT = resolve(import.meta.dirname, '..', '..');
+const CLI = join(PROJECT_ROOT, 'src', 'index.ts');
+
 describe('Completion Module', () => {
   let consoleLogSpy: any[];
   let consoleErrorSpy: any[];
   let originalLog: typeof console.log;
   let originalError: typeof console.error;
-  let processExitStub: ((code: number) => never) | undefined;
-  let exitCode: number | undefined;
 
   beforeEach(() => {
     consoleLogSpy = [];
     consoleErrorSpy = [];
     originalLog = console.log;
     originalError = console.error;
-    exitCode = undefined;
 
     console.log = (...args: any[]) => {
       consoleLogSpy.push(args.join(' '));
@@ -28,21 +30,11 @@ describe('Completion Module', () => {
     console.error = (...args: any[]) => {
       consoleErrorSpy.push(args.join(' '));
     };
-
-    // Stub process.exit to capture exit codes
-    processExitStub = process.exit;
-    (process.exit as any) = (code: number) => {
-      exitCode = code;
-      throw new Error(`process.exit(${code})`);
-    };
   });
 
   afterEach(() => {
     console.log = originalLog;
     console.error = originalError;
-    if (processExitStub) {
-      process.exit = processExitStub;
-    }
   });
 
   test('generates bash completion script', () => {
@@ -119,36 +111,28 @@ describe('Completion Module', () => {
     assert.ok(output.includes('-l profile'));
   });
 
+  // Error cases: run the CLI as a subprocess so that process.exit happens
+  // in a child process and does not interfere with the test runner.
+
   test('handles missing shell argument', () => {
-    try {
-      showCompletion(undefined);
-      assert.fail('Should have thrown an error');
-    } catch (error: any) {
-      assert.ok(error.message.includes('process.exit(1)'));
-      assert.strictEqual(exitCode, 1);
-    }
-
-    const errorOutput = consoleErrorSpy.join('\n');
-    assert.ok(errorOutput.includes('Shell type required'));
-    assert.ok(errorOutput.includes('c8 completion <bash|zsh|fish>'));
+    const result = spawnSync('node', [CLI, 'completion'], {
+      encoding: 'utf-8',
+      cwd: PROJECT_ROOT,
+    });
+    assert.strictEqual(result.status, 1, 'Should exit with code 1');
+    assert.ok(result.stderr.includes('Shell type required'), `stderr should mention "Shell type required". Got: ${result.stderr}`);
+    assert.ok(result.stderr.includes('c8 completion <bash|zsh|fish>'), `stderr should include usage hint. Got: ${result.stderr}`);
   });
 
   test('handles unknown shell', () => {
-    try {
-      showCompletion('powershell');
-      assert.fail('Should have thrown an error');
-    } catch (error: any) {
-      assert.ok(error.message.includes('process.exit(1)'));
-      assert.strictEqual(exitCode, 1);
-    }
-
-    // logger.error outputs to console.error
-    const errorOutput = consoleErrorSpy.join('\n');
-    assert.ok(errorOutput.includes('Unknown shell: powershell'));
-
-    // logger.info outputs to console.log
-    const logOutput = consoleLogSpy.join('\n');
-    assert.ok(logOutput.includes('Supported shells: bash, zsh, fish'));
+    const result = spawnSync('node', [CLI, 'completion', 'powershell'], {
+      encoding: 'utf-8',
+      cwd: PROJECT_ROOT,
+    });
+    assert.strictEqual(result.status, 1, 'Should exit with code 1');
+    assert.ok(result.stderr.includes('Unknown shell: powershell'), `stderr should mention "Unknown shell: powershell". Got: ${result.stderr}`);
+    // logger.info() in text mode goes to stdout
+    assert.ok(result.stdout.includes('Supported shells: bash, zsh, fish'), `stdout should list supported shells. Got: ${result.stdout}`);
   });
 
   test('handles case-insensitive shell names', () => {