fix(runner): keep the may-still-be-executing warning on the unreachable snippet answer

Author: felipe-augusto

.changeset/evaluate-snippet-failure-reasons.md

@@ -2,4 +2,4 @@
 "@qawolf/cli": patch
 ---
 
-Tell a runner that cannot evaluate snippets apart from one that could not be reached, so the message names which it is and only the reachable case suggests retrying.
+`qawolf runner exec` now tells apart a runner with nothing attached to evaluate a snippet (`runner-cannot-evaluate-snippets`, exit `2`, which will never clear) from one that could not be reached (`runner-unreachable`, exit `4`, which may still be starting or busy) instead of reporting both the same way. Only the unreachable message warns that the snippet may still be executing, since only that case can have taken effect before its answer was lost.

src/core/messages/interactiveRunner/interact.ts

@@ -18,10 +18,6 @@ export const interactMessages = {
     `The runner answered "${failureReason}", which this version of the CLI does not know how to report. Upgrade with npm install -g @qawolf/cli.`,
   runnerHasNoScreen:
     "This runner does not run a browser on a virtual desktop, so there is nothing about it to see or drive. Retrying will never help: launch a playwright runner instead.",
-  runnerHasNoScreenToEvaluate:
-    "The runner has no live page to evaluate the snippet against: a freshly launched runner has no page until a run opens one, so run a flow on it with qawolf runner run first. If it is not a runner that runs a browser at all, this will never clear. It is not proof the snippet did not run, so do not resubmit one that mutates the page without reading qawolf runner events console first.",
-  evaluateAnsweredUnknown: (failureReason: string) =>
-    `The runner answered "${failureReason}", which this version of the CLI does not know how to report. Upgrade with npm install -g @qawolf/cli.`,
   inspectNeedsABrowserRunner:
     "This runner is a mobile device, and element and page HTML are a browser's shapes. Retrying will never help. Launch a playwright runner to inspect one.",
   nothingToInspect: (errorMessage: string | undefined) =>
@@ -50,8 +46,16 @@ export const interactMessages = {
     `Could not read "${path}". Name a readable file, or "-" to read the snippet from stdin.`,
   snippetRan:
     "The snippet ran. Its value is not returned: read anything it printed with qawolf runner events console.",
+  // Alone among these failure reasons, this one may have taken effect before its
+  // answer was lost, so the message must not invite a bare repeat.
+  snippetRunnerUnreachable:
+    "The runner could not be reached. It may still be starting, or it may have terminated after inactivity. It is not proof the snippet did not run: a snippet that outlives the answer window is still executing when you read this, so do not blindly resubmit one that mutates the page without reading qawolf runner events console first.",
   snippetStopped:
     "The snippet was interrupted before it finished. Anything it printed first is in qawolf runner events console.",
+  runnerCannotEvaluateSnippets:
+    "This runner has no snippet evaluator attached, so there is nothing to run code against. Retrying will never help: launch a playwright, android, ios, or windows runner instead.",
+  snippetAnsweredUnknown: (failureReason: string) =>
+    `The runner answered "${failureReason}", which this version of the CLI does not know how to report. Upgrade with npm install -g @qawolf/cli.`,
   stdinEmptyAction:
     'Nothing arrived on stdin. Pipe the action in, or name the action type instead of "-".',
   stdinEmptySnippet:

src/domains/interactiveRunner/evaluateSnippet.test.ts

@@ -218,4 +218,27 @@ describe("handleRunnerExec", () => {
     expect(result?.errorBody).toContain("quota reached");
     expect(result?.exitCode).toBe(4);
   });
+
+  // A runner with nothing attached to serve `exec` will never clear, unlike a
+  // genuinely unreachable one, which may still have run the snippet before its
+  // answer was lost.
+  it.each([
+    ["runner-cannot-evaluate-snippets", 2, "no snippet evaluator attached"],
+    ["runner-unreachable", 4, "not proof the snippet did not run"],
+  ])("reports %s", async (failureReason, exitCode, text) => {
+    const { callPublicApi, ctx } = makeAuthCtx();
+    callPublicApi.mockResolvedValue({
+      ok: true,
+      value: { failureReason, outcome: "failure" },
+    });
+
+    const result = await handleRunnerExec(
+      ctx,
+      { contextFile: undefined, runner: "ci", source: "flow.ts" },
+      makeTestDeps(),
+    );
+
+    expect(result?.exitCode).toBe(exitCode);
+    expect(result?.error).toContain(text);
+  });
 });

src/domains/interactiveRunner/evaluateSnippetFailure.test.ts

@@ -2,6 +2,7 @@ import type { publicContractsV1 } from "@qawolf/api-contracts/v1";
 import type { z } from "zod";
 
 import { interactiveRunnerMessages } from "~/core/messages/index.js";
+import type { CommandResult } from "~/shell/commandContext.js";
 import { exitCodes } from "~/shell/exit.js";
 
 type EvaluateSnippetFailure = Extract<
@@ -10,30 +11,28 @@ type EvaluateSnippetFailure = Extract<
 >;
 
 /** Why a snippet was not evaluated, and what the caller should do about it. */
-export type EvaluateSnippetRefusal = {
-  error: string;
-  exitCode: number;
-};
-
 export function describeEvaluateSnippetFailure(
   failure: EvaluateSnippetFailure,
-): EvaluateSnippetRefusal {
+): Exclude<CommandResult, void> {
   const { failureReason } = failure;
   switch (failureReason) {
     case "runner-cannot-evaluate-snippets":
       return {
-        error: interactiveRunnerMessages.runnerHasNoScreenToEvaluate,
+        error: interactiveRunnerMessages.runnerCannotEvaluateSnippets,
         exitCode: exitCodes.invalidArgs,
       };
+    // Alone among these failure reasons, this one may have taken effect
+    // before its answer was lost, so the message must not invite a bare
+    // repeat.
     case "runner-unreachable":
       return {
-        error: interactiveRunnerMessages.runnerUnreachable,
+        error: interactiveRunnerMessages.snippetRunnerUnreachable,
         exitCode: exitCodes.network,
       };
     default:
       failureReason satisfies never;
       return {
-        error: interactiveRunnerMessages.evaluateAnsweredUnknown(failureReason),
+        error: interactiveRunnerMessages.snippetAnsweredUnknown(failureReason),
         exitCode: exitCodes.network,
       };
   }