fix(vitest-runner): never report a mutant run that executed zero tests as survived

packages/vitest-runner/src/vitest-test-runner.ts

@@ -50,6 +50,12 @@ type StrykerNamespace = '__stryker__' | '__stryker2__';
 const STRYKER_SETUP = fileURLToPath(
   new URL('./stryker-setup.js', import.meta.url),
 );
+/**
+ * Maximum number of times a mutant run is re-issued when vitest completed it
+ * without executing any of the tests selected by the test filter. See
+ * https://github.com/stryker-mutator/stryker-js/issues/6073.
+ */
+const MAX_EMPTY_FILTERED_RUN_RETRIES = 3;
 
 interface RunFilter {
   /**
@@ -207,15 +213,51 @@ export class VitestTestRunner implements TestRunner {
     this.ctx!.provide('hitLimit', options.hitLimit);
     this.ctx!.provide('mutantActivation', options.mutantActivation);
     this.ctx!.provide('activeMutant', options.activeMutant.id);
-    const dryRunResult = await this.run({
+    const runFilter: RunFilter = {
       testIds: options.testFilter,
       relatedFiles: [options.sandboxFileName],
-    });
+    };
+    let dryRunResult = await this.run(runFilter);
+    for (
+      let retry = 1;
+      this.isEmptyFilteredRun(options.testFilter, dryRunResult) &&
+      retry <= MAX_EMPTY_FILTERED_RUN_RETRIES;
+      retry++
+    ) {
+      this.log.debug(
+        'Mutant run for mutant %s completed without executing any of its %s filtered tests, retrying (%s/%s).',
+        options.activeMutant.id,
+        options.testFilter!.length,
+        retry,
+        MAX_EMPTY_FILTERED_RUN_RETRIES,
+      );
+      dryRunResult = await this.run(runFilter);
+    }
+    if (this.isEmptyFilteredRun(options.testFilter, dryRunResult)) {
+      // A run that executed none of its selected tests proves nothing about
+      // the mutant; reporting it based on "no test failed" would produce a
+      // false survivor (see https://github.com/stryker-mutator/stryker-js/issues/6073).
+      return toMutantRunResult({
+        status: DryRunStatus.Error,
+        errorMessage: `Vitest completed the mutant run without executing any of the ${options.testFilter!.length} tests selected for mutant ${options.activeMutant.id} (after ${MAX_EMPTY_FILTERED_RUN_RETRIES} retries).`,
+      });
+    }
     const hitCount = this.readHitCount();
     const timeOut = determineHitLimitReached(hitCount, options.hitLimit);
     return toMutantRunResult(timeOut ?? dryRunResult);
   }
 
+  private isEmptyFilteredRun(
+    testFilter: string[] | undefined,
+    result: DryRunResult,
+  ): boolean {
+    return (
+      (testFilter?.length ?? 0) > 0 &&
+      result.status === DryRunStatus.Complete &&
+      result.tests.length === 0
+    );
+  }
+
   private async run({
     testIds = [],
     relatedFiles,

packages/vitest-runner/test/unit/vitest-runner.spec.ts

@@ -3,15 +3,20 @@ import { expect } from 'chai';
 import fs from 'fs';
 import { factory, testInjector } from '@stryker-mutator/test-helpers';
 import {
+  ErrorMutantRunResult,
   MutantRunStatus,
   TestRunnerCapabilities,
 } from '@stryker-mutator/api/test-runner';
-import { Vitest } from 'vitest/node';
+import { RunnerTestFile, Vitest } from 'vitest/node';
 
 import { VitestTestRunner } from '../../src/vitest-test-runner.js';
 import { VitestRunnerOptionsWithStrykerOptions } from '../../src/vitest-runner-options-with-stryker-options.js';
 import { vitestWrapper } from '../../src/vitest-wrapper.js';
-import { createVitestMock } from '../util/factories.js';
+import {
+  createVitestFile,
+  createVitestMock,
+  createVitestTest,
+} from '../util/factories.js';
 import { VITEST_ERROR_CODES } from '../../src/vitest-helpers.js';
 
 describe(VitestTestRunner.name, () => {
@@ -156,6 +161,55 @@ describe(VitestTestRunner.name, () => {
         'mutant',
       );
     });
+
+    it('should report an error instead of survived when a filtered run repeatedly executes no tests', async () => {
+      // A run that executed none of the selected tests proves nothing about
+      // the mutant; treating it as "no test failed" produces false survivors.
+      // See https://github.com/stryker-mutator/stryker-js/issues/6073
+      const options = factory.mutantRunOptions({
+        testFilter: ['file.spec.js#suite-test > test1'],
+        activeMutant: factory.mutant({ id: '42' }),
+      });
+
+      const result = await sut.mutantRun(options);
+
+      expect(result.status).eq(MutantRunStatus.Error);
+      expect((result as ErrorMutantRunResult).errorMessage).contains(
+        'without executing any of the 1 tests selected for mutant 42',
+      );
+      // initial run + 3 retries
+      sinon.assert.callCount(vitestStub.start as sinon.SinonStub, 4);
+    });
+
+    it('should return the verdict of a retried filtered run that does execute tests', async () => {
+      const passingFile = createVitestFile({
+        tasks: [createVitestTest({ result: { state: 'pass', duration: 1 } })],
+      });
+      const getFilesStub = sinon.stub<[], RunnerTestFile[]>();
+      getFilesStub.onFirstCall().returns([]);
+      getFilesStub.returns([passingFile]);
+      (
+        vitestStub.state as unknown as { getFiles: () => RunnerTestFile[] }
+      ).getFiles = getFilesStub;
+      const options = factory.mutantRunOptions({
+        testFilter: ['file.spec.js#suite-test > test1'],
+      });
+
+      const result = await sut.mutantRun(options);
+
+      expect(result.status).eq(MutantRunStatus.Survived);
+      // initial run + 1 retry
+      sinon.assert.callCount(vitestStub.start as sinon.SinonStub, 2);
+    });
+
+    it('should not treat an unfiltered run without tests as an error', async () => {
+      const result = await sut.mutantRun(
+        factory.mutantRunOptions({ testFilter: undefined }),
+      );
+
+      expect(result.status).eq(MutantRunStatus.Survived);
+      sinon.assert.callCount(vitestStub.start as sinon.SinonStub, 1);
+    });
   });
 
   describe(VitestTestRunner.prototype.dryRun.name, () => {