fix(runner): report a runner that cannot evaluate snippets, not unreachable (ARC-610)

.changeset/snippet-cannot-evaluate-refusal.md

@@ -0,0 +1,7 @@
+---
+"@qawolf/cli": patch
+---
+
+`qawolf runner exec` now tells apart a runner with nothing attached to evaluate a snippet (`runner-cannot-evaluate-snippets`, which will never clear) from one that could not be reached (`runner-unreachable`, which may still be starting or busy) instead of reporting both the same way.
+
+This depends on `@qawolf/api-contracts` publishing the `runner-cannot-evaluate-snippets` failure reason on `runner.evaluateSnippet` (qawolf/platform#32612, ARC-610); it ships once that dependency is bumped in a preceding release.

src/core/messages/interactiveRunner/interact.ts

@@ -18,8 +18,6 @@ export const interactMessages = {
     `The runner answered "${failureReason}", which this version of the CLI does not know how to report. Upgrade with npm install -g @qawolf/cli.`,
   runnerHasNoScreen:
     "This runner does not run a browser on a virtual desktop, so there is nothing about it to see or drive. Retrying will never help: launch a playwright runner instead.",
-  runnerHasNoScreenToEvaluate:
-    "The runner could not evaluate the snippet. This covers a runner that is still starting or busy, and also one with no live page to evaluate against: a freshly launched runner has no page until a run opens one, so run a flow on it with qawolf runner run first. If it is not a runner that runs a browser at all, this will never clear. It is not proof the snippet did not run, so do not resubmit one that mutates the page without reading qawolf runner events console first.",
   inspectNeedsABrowserRunner:
     "This runner is a mobile device, and element and page HTML are a browser's shapes. Retrying will never help. Launch a playwright runner to inspect one.",
   nothingToInspect: (errorMessage: string | undefined) =>
@@ -48,8 +46,16 @@ export const interactMessages = {
     `Could not read "${path}". Name a readable file, or "-" to read the snippet from stdin.`,
   snippetRan:
     "The snippet ran. Its value is not returned: read anything it printed with qawolf runner events console.",
+  // Alone among these failure reasons, this one may have taken effect before its
+  // answer was lost, so the message must not invite a bare repeat.
+  snippetRunnerUnreachable:
+    "The runner could not be reached. It may still be starting, or it may have terminated after inactivity. It is not proof the snippet did not run: a snippet that outlives the answer window is still executing when you read this, so do not blindly resubmit one that mutates the page without reading qawolf runner events console first.",
   snippetStopped:
     "The snippet was interrupted before it finished. Anything it printed first is in qawolf runner events console.",
+  runnerCannotEvaluateSnippets:
+    "This runner has no snippet evaluator attached, so there is nothing to run code against. Retrying will never help: launch a playwright, android, ios, or windows runner instead.",
+  snippetAnsweredUnknown: (failureReason: string) =>
+    `The runner answered "${failureReason}", which this version of the CLI does not know how to report. Upgrade with npm install -g @qawolf/cli.`,
   stdinEmptyAction:
     'Nothing arrived on stdin. Pipe the action in, or name the action type instead of "-".',
   stdinEmptySnippet:

src/domains/interactiveRunner/evaluateSnippet.test.ts

@@ -219,13 +219,17 @@ describe("handleRunnerExec", () => {
     expect(result?.exitCode).toBe(4);
   });
 
-  // For this verb, unreachable also covers a runner with no live page, which will
-  // never clear, so the message has to name that rather than just say "retry".
-  it("names the no-live-page case an unreachable answer hides", async () => {
+  // A runner with nothing attached to serve `exec` will never clear, unlike a
+  // genuinely unreachable one, which may still have run the snippet before its
+  // answer was lost.
+  it.each([
+    ["runner-cannot-evaluate-snippets", 2, "no snippet evaluator attached"],
+    ["runner-unreachable", 4, "not proof the snippet did not run"],
+  ])("reports %s", async (failureReason, exitCode, text) => {
     const { callPublicApi, ctx } = makeAuthCtx();
     callPublicApi.mockResolvedValue({
       ok: true,
-      value: { failureReason: "runner-unreachable", outcome: "failure" },
+      value: { failureReason, outcome: "failure" },
     });
 
     const result = await handleRunnerExec(
@@ -234,11 +238,7 @@ describe("handleRunnerExec", () => {
       makeTestDeps(),
     );
 
-    expect(result?.error).toContain("no live page");
-    // A fresh playwright runner does run a browser, so telling the
-    // caller to check the image would send them after the wrong thing.
-    expect(result?.error).toContain("run a flow on it with qawolf runner run");
-    expect(result?.error).toContain("not proof the snippet did not run");
-    expect(result?.exitCode).toBe(4);
+    expect(result?.exitCode).toBe(exitCode);
+    expect(result?.error).toContain(text);
   });
 });

src/domains/interactiveRunner/evaluateSnippet.ts

@@ -14,6 +14,7 @@ import { exitCodes } from "~/shell/exit.js";
 import { failureFields } from "~/shell/platform/requestWithRetry.js";
 
 import type { InteractiveRunnerDeps } from "./deps.js";
+import { describeEvaluateSnippetFailure } from "./evaluateSnippetFailure.js";
 import { announceRunner, resolveRunner } from "./resolveRunner.js";
 import { runnerCallOptions } from "./runnerCallOptions.js";
 
@@ -115,11 +116,7 @@ export async function handleRunnerExec(
   }
 
   if (result.value.outcome === "failure") {
-    result.value.failureReason satisfies "runner-unreachable";
-    return {
-      error: interactiveRunnerMessages.runnerHasNoScreenToEvaluate,
-      exitCode: exitCodes.network,
-    };
+    return describeEvaluateSnippetFailure(result.value);
   }
 
   switch (result.value.result) {

src/domains/interactiveRunner/evaluateSnippetFailure.ts

@@ -0,0 +1,39 @@
+import type { publicContractsV1 } from "@qawolf/api-contracts/v1";
+import type { z } from "zod";
+
+import { interactiveRunnerMessages } from "~/core/messages/index.js";
+import type { CommandResult } from "~/shell/commandContext.js";
+import { exitCodes } from "~/shell/exit.js";
+
+type EvaluateSnippetFailure = Extract<
+  z.output<typeof publicContractsV1.runner.evaluateSnippet.output>,
+  { outcome: "failure" }
+>;
+
+/** Why a snippet was not evaluated, and what the caller should do about it. */
+export function describeEvaluateSnippetFailure(
+  failure: EvaluateSnippetFailure,
+): Exclude<CommandResult, void> {
+  const { failureReason } = failure;
+  switch (failureReason) {
+    case "runner-cannot-evaluate-snippets":
+      return {
+        error: interactiveRunnerMessages.runnerCannotEvaluateSnippets,
+        exitCode: exitCodes.invalidArgs,
+      };
+    // Alone among these failure reasons, this one may have taken effect
+    // before its answer was lost, so the message must not invite a bare
+    // repeat.
+    case "runner-unreachable":
+      return {
+        error: interactiveRunnerMessages.snippetRunnerUnreachable,
+        exitCode: exitCodes.network,
+      };
+    default:
+      failureReason satisfies never;
+      return {
+        error: interactiveRunnerMessages.snippetAnsweredUnknown(failureReason),
+        exitCode: exitCodes.network,
+      };
+  }
+}