prevent execution queue being blocked by zombie processes (#1134)

Author: connectdotz

Diff for: src/JestExt/process-listeners.ts

@@ -1,7 +1,7 @@
 import * as vscode from 'vscode';
 import { JestTotalResults, RunnerEvent } from 'jest-editor-support';
 import { cleanAnsi, toErrorString } from '../helpers';
-import { JestProcess } from '../JestProcessManagement';
+import { JestProcess, ProcessStatus } from '../JestProcessManagement';
 import { ListenerSession, ListTestFilesCallback } from './process-session';
 import { Logging } from '../logging';
 import { JestRunEvent } from './types';
@@ -279,10 +279,7 @@ export class RunTestListener extends AbstractProcessListener {
 
   // watch process should not exit unless we request it to be closed
   private handleWatchProcessCrash(process: JestProcess): string | undefined {
-    if (
-      (process.request.type === 'watch-tests' || process.request.type === 'watch-all-tests') &&
-      process.stopReason !== 'on-demand'
-    ) {
+    if (process.isWatchMode && process.status !== ProcessStatus.Cancelled) {
       const msg = `Jest process "${process.request.type}" ended unexpectedly`;
       this.logging('warn', msg);
 
@@ -354,16 +351,19 @@ export class RunTestListener extends AbstractProcessListener {
 
   protected onProcessClose(process: JestProcess, code?: number, signal?: string): void {
     this.runEnded();
-    let error = this.handleWatchProcessCrash(process);
+    let error: string | undefined;
 
-    if (code && code > 1) {
-      if (this.retryWithLoginShell(process, code, signal)) {
-        return;
-      }
-      if (!error) {
+    if (process.status !== ProcessStatus.Cancelled) {
+      if (code && code > 1) {
+        if (this.retryWithLoginShell(process, code, signal)) {
+          return;
+        }
         error = `process ${process.id} exited with code= ${code}`;
       }
+
+      error = this.handleWatchProcessCrash(process) ?? error;
     }
+
     this.onRunEvent.fire({ type: 'exit', process, error, code });
   }
 }

Diff for: src/JestProcessManagement/JestProcess.ts

@@ -4,7 +4,7 @@ import { Runner, RunnerEvent, Options } from 'jest-editor-support';
 import { JestExtContext, WatchMode } from '../JestExt/types';
 import { extensionId } from '../appGlobals';
 import { Logging } from '../logging';
-import { JestProcessInfo, JestProcessRequest, UserDataType } from './types';
+import { JestProcessInfo, JestProcessRequest, ProcessStatus, UserDataType } from './types';
 import { requestString } from './helper';
 import { toFilePath, removeSurroundingQuote, escapeRegExp, shellQuote } from '../helpers';
 
@@ -23,20 +23,18 @@ interface RunnerTask {
   reject: (reason: unknown) => unknown;
   runner: Runner;
 }
-export type StopReason = 'on-demand' | 'process-end';
 
 let SEQ = 0;
 
 export class JestProcess implements JestProcessInfo {
-  static readonly stopHangTimeout = 500;
-
   private task?: RunnerTask;
   private extContext: JestExtContext;
   private logging: Logging;
-  private _stopReason?: StopReason;
   public readonly id: string;
   private desc: string;
   public readonly request: JestProcessRequest;
+  public _status: ProcessStatus;
+  private autoStopTimer?: NodeJS.Timeout;
 
   constructor(
     extContext: JestExtContext,
@@ -48,10 +46,11 @@ export class JestProcess implements JestProcessInfo {
     this.logging = extContext.loggingFactory.create(`JestProcess ${request.type}`);
     this.id = `${request.type}-${SEQ++}`;
     this.desc = `id: ${this.id}, request: ${requestString(request)}`;
+    this._status = ProcessStatus.Pending;
   }
 
-  public get stopReason(): StopReason | undefined {
-    return this._stopReason;
+  public get status(): ProcessStatus {
+    return this._status;
   }
 
   private get watchMode(): WatchMode {
@@ -64,15 +63,39 @@ export class JestProcess implements JestProcessInfo {
     return WatchMode.None;
   }
 
+  public get isWatchMode(): boolean {
+    return this.watchMode !== WatchMode.None;
+  }
+
   public toString(): string {
-    return `JestProcess: ${this.desc}; stopReason: ${this.stopReason}`;
+    return `JestProcess: ${this.desc}; status: "${this.status}"`;
   }
-  public start(): Promise<void> {
-    this._stopReason = undefined;
-    return this.startRunner();
+
+  /**
+   * To prevent zombie process, this method will automatically stops the Jest process if it is running for too long. The process will be marked as "Cancelled" and stopped.
+   * Warning: This should only be called when you are certain the process should end soon, for example a non-watch mode process should end after the test results have been processed.
+   * @param delay The delay in milliseconds after which the process will be considered hung and stopped. Default is 30000 milliseconds (30 seconds ).
+   */
+  public autoStop(delay = 30000, onStop?: (process: JestProcessInfo) => void): void {
+    if (this.status === ProcessStatus.Running) {
+      if (this.autoStopTimer) {
+        clearTimeout(this.autoStopTimer);
+      }
+      this.autoStopTimer = setTimeout(() => {
+        if (this.status === ProcessStatus.Running) {
+          console.warn(
+            `Jest Process "${this.id}": will be force closed due to the autoStop Timer (${delay} msec) `
+          );
+          this.stop();
+          onStop?.(this);
+        }
+      }, delay);
+    }
   }
+
   public stop(): Promise<void> {
-    this._stopReason = 'on-demand';
+    this._status = ProcessStatus.Cancelled;
+
     if (!this.task) {
       this.logging('debug', 'nothing to stop, no pending runner/promise');
       this.taskDone();
@@ -99,12 +122,19 @@ export class JestProcess implements JestProcessInfo {
     return `"${removeSurroundingQuote(aString)}"`;
   }
 
-  private startRunner(): Promise<void> {
+  public start(): Promise<void> {
+    if (this.status === ProcessStatus.Cancelled) {
+      this.logging('warn', `the runner task has been cancelled!`);
+      return Promise.resolve();
+    }
+
     if (this.task) {
       this.logging('warn', `the runner task has already started!`);
       return this.task.promise;
     }
 
+    this._status = ProcessStatus.Running;
+
     const options: Options = {
       noColor: false,
       reporters: ['default', `"${this.getReporterPath()}"`],
@@ -196,7 +226,13 @@ export class JestProcess implements JestProcessInfo {
     if (event === 'processClose' || event === 'processExit') {
       this.task?.resolve();
       this.task = undefined;
-      this._stopReason = this._stopReason ?? 'process-end';
+
+      clearTimeout(this.autoStopTimer);
+      this.autoStopTimer = undefined;
+
+      if (this._status !== ProcessStatus.Cancelled) {
+        this._status = ProcessStatus.Done;
+      }
     }
     this.request.listener.onEvent(this, event, ...args);
   }

Diff for: src/JestProcessManagement/JestProcessManager.ts

@@ -6,6 +6,7 @@ import {
   Task,
   JestProcessInfo,
   UserDataType,
+  ProcessStatus,
 } from './types';
 import { Logging } from '../logging';
 import { createTaskQueue, TaskQueue } from './task-queue';
@@ -78,11 +79,13 @@ export class JestProcessManager implements TaskArrayFunctions<JestProcess> {
       return;
     }
     const process = task.data;
-
     try {
-      const promise = process.start();
-      this.extContext.onRunEvent.fire({ type: 'process-start', process });
-      await promise;
+      // process could be cancelled before it starts, so check before starting
+      if (process.status === ProcessStatus.Pending) {
+        const promise = process.start();
+        this.extContext.onRunEvent.fire({ type: 'process-start', process });
+        await promise;
+      }
     } catch (e) {
       this.logging('error', `${queue.name}: process failed to start:`, process, e);
       this.extContext.onRunEvent.fire({

Diff for: src/JestProcessManagement/types.ts

@@ -15,12 +15,25 @@ export interface UserDataType {
   testError?: boolean;
   testItem?: vscode.TestItem;
 }
+export enum ProcessStatus {
+  Pending = 'pending',
+  Running = 'running',
+  Cancelled = 'cancelled',
+  // process exited not because of cancellation
+  Done = 'done',
+}
+
 export interface JestProcessInfo {
   readonly id: string;
   readonly request: JestProcessRequest;
   // user data is a way to store data that is outside of the process managed by the processManager.
   // subsequent use of this data is up to the user but should be aware that multiple components might contribute to this data.
   userData?: UserDataType;
+  stop: () => Promise<void>;
+  status: ProcessStatus;
+  isWatchMode: boolean;
+  // starting a timer to automatically kill the process after x milliseconds if the process is still running.
+  autoStop: (delay?: number, onStop?: (process: JestProcessInfo) => void) => void;
 }
 
 export type TaskStatus = 'running' | 'pending';