Collapse built-in tool list in replay-proxy snapshot matching (#2013)

When a model calls a nonexistent tool, the runtime replies with
"Available tools that can be called are <list>." The E2E replay proxy
embedded that literal enumeration in ~54 snapshots, so any change to the
built-in tool set (e.g. the new write_agent tool) broke snapshot matching
and caused the copilot-agent-runtime C# SDK canary to retry forever until
the 45m timeout.

Collapse the entire enumeration to a stable ${available_tools} placeholder
on both the live-request side (normalizeAvailableToolNames) and the stored
snapshot side (new normalizeStoredToolMessages applied at load time), so
snapshots keep matching as the built-in tool set evolves across runtime
versions and platforms.


Copilot-Session: fa431f41-c7dc-4580-9cba-274170ee64df

Co-authored-by: TestUser <test@example.com>
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: 3 people

test/harness/replayingCapiProxy.test.ts

@@ -509,6 +509,47 @@ Always include PINEAPPLE_COCONUT_42.
     expect(toolMessages[1].content).toBe("[beta result]");
   });
 
+  test("collapses the available-tools list to a stable placeholder", async () => {
+    const requestBody = JSON.stringify({
+      messages: [
+        { role: "user", content: "Help me" },
+        {
+          role: "assistant",
+          tool_calls: [
+            {
+              id: "tc1",
+              type: "function",
+              function: { name: "report_intent", arguments: "{}" },
+            },
+          ],
+        },
+        {
+          role: "tool",
+          tool_call_id: "tc1",
+          content:
+            "Tool 'report_intent' does not exist. Available tools that can be called are bash, read_bash, view, read_agent, list_agents, write_agent, grep, glob, task.",
+        },
+      ],
+    });
+    const responseBody = JSON.stringify({
+      choices: [{ message: { role: "assistant", content: "Done" } }],
+    });
+
+    const outputPath = await createProxy([
+      { url: "/chat/completions", requestBody, responseBody },
+    ]);
+
+    const result = await readYamlOutput(outputPath);
+    const toolMessage = result.conversations[0].messages.find(
+      (m) => m.role === "tool",
+    );
+    // The whole enumeration collapses so snapshots stay stable as the built-in
+    // tool set evolves (e.g. write_agent being added).
+    expect(toolMessage?.content).toBe(
+      "Tool 'report_intent' does not exist. Available tools that can be called are ${available_tools}.",
+    );
+  });
+
   test("normalizes read_agent timing metadata", async () => {
     const requestBody = JSON.stringify({
       messages: [
@@ -827,6 +868,85 @@ Always include PINEAPPLE_COCONUT_42.
       }
     });
 
+    test("matches available-tools results after the built-in tool set changes", async () => {
+      const cachePath = path.join(tempDir, "cache.yaml");
+      // Legacy snapshot recorded before write_agent was a built-in tool: the
+      // enumeration frozen on disk still contains the older tool list.
+      const cacheContent = yaml.stringify({
+        models: ["test-model"],
+        conversations: [
+          {
+            messages: [
+              { role: "system", content: "${system}" },
+              { role: "user", content: "Report intent" },
+              {
+                role: "assistant",
+                tool_calls: [
+                  {
+                    id: "toolcall_0",
+                    type: "function",
+                    function: { name: "report_intent", arguments: "{}" },
+                  },
+                ],
+              },
+              {
+                role: "tool",
+                tool_call_id: "toolcall_0",
+                content:
+                  "Tool 'report_intent' does not exist. Available tools that can be called are ${shell}, view, read_agent, list_agents, grep, glob, task.",
+              },
+              { role: "assistant", content: "Done" },
+            ],
+          },
+        ],
+      } satisfies NormalizedData);
+      await writeFile(cachePath, cacheContent);
+
+      const proxy = new ReplayingCapiProxy(
+        "http://localhost:9999",
+        cachePath,
+        workDir,
+      );
+      const proxyUrl = await proxy.start();
+
+      try {
+        const response = await makeRequest(proxyUrl, "/chat/completions", {
+          body: {
+            model: "test-model",
+            messages: [
+              { role: "system", content: "System prompt" },
+              { role: "user", content: "Report intent" },
+              {
+                role: "assistant",
+                tool_calls: [
+                  {
+                    id: "runtime-call-id",
+                    type: "function",
+                    function: { name: "report_intent", arguments: "{}" },
+                  },
+                ],
+              },
+              {
+                role: "tool",
+                tool_call_id: "runtime-call-id",
+                // Newer runtime added write_agent to the built-in tool set.
+                content:
+                  "Tool 'report_intent' does not exist. Available tools that can be called are bash, read_bash, view, read_agent, list_agents, write_agent, grep, glob, task.",
+              },
+            ],
+          },
+        });
+
+        expect(response.status).toBe(200);
+        expect(
+          (JSON.parse(response.body) as ChatCompletion).choices[0].message
+            .content,
+        ).toBe("Done");
+      } finally {
+        await proxy.stop();
+      }
+    });
+
     test("expands workdir placeholder in cached response", async () => {
       const cachePath = path.join(tempDir, "cache.yaml");
       const cacheContent = yaml.stringify({

test/harness/replayingCapiProxy.ts

@@ -133,6 +133,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy {
       this.state.storedData = yaml.parse(content) as NormalizedData;
       normalizeToolResultOrder(this.state.storedData.conversations);
       normalizeStoredUserMessages(this.state.storedData.conversations);
+      normalizeStoredToolMessages(this.state.storedData.conversations);
     }
   }
 
@@ -1079,6 +1080,22 @@ function normalizeStoredUserMessages(conversations: NormalizedConversation[]) {
   }
 }
 
+// Re-normalizes the built-in tool enumeration in stored tool results at load
+// time. Snapshots recorded before normalizeAvailableToolNames collapsed the
+// whole list (or recorded against an older tool set) still contain the literal
+// enumeration on disk; the result normalizers only run against live requests,
+// so without this the stored side would keep the stale list and never match a
+// request whose tool set has since changed.
+function normalizeStoredToolMessages(conversations: NormalizedConversation[]) {
+  for (const conversation of conversations) {
+    for (const message of conversation.messages) {
+      if (message.role === "tool" && typeof message.content === "string") {
+        message.content = normalizeAvailableToolNames(message.content);
+      }
+    }
+  }
+}
+
 function normalizeSkillContextFrontmatter(content: string): string {
   // Runtime versions may include or omit SKILL.md metadata in the prompt context.
   return content.replace(
@@ -1169,41 +1186,21 @@ function normalizeReadAgentTimings(result: string): string {
     .replace(/\bduration: \d+(?:\.\d+)?s\b/g, "duration: 0s");
 }
 
-// Maps the platform-specific shell tool family names to stable placeholders.
-// On Windows the runtime exposes powershell/read_powershell/stop_powershell/...,
-// on Linux/macOS it exposes bash/read_bash/stop_bash/.... Ordered so that the
-// prefixed names are handled explicitly; \b boundaries keep bare names from
-// matching inside the prefixed ones.
-const shellToolFamilyReplacements: ReadonlyArray<readonly [RegExp, string]> = [
-  [/\bread_powershell\b/g, "${read_shell}"],
-  [/\bstop_powershell\b/g, "${stop_shell}"],
-  [/\blist_powershell\b/g, "${list_shell}"],
-  [/\bwrite_powershell\b/g, "${write_shell}"],
-  [/\bpowershell\b/g, "${shell}"],
-  [/\bread_bash\b/g, "${read_shell}"],
-  [/\bstop_bash\b/g, "${stop_shell}"],
-  [/\blist_bash\b/g, "${list_shell}"],
-  [/\bwrite_bash\b/g, "${write_shell}"],
-  [/\bbash\b/g, "${shell}"],
-];
-
-function normalizeShellToolFamilyNames(text: string): string {
-  let result = text;
-  for (const [pattern, replacement] of shellToolFamilyReplacements) {
-    result = result.replace(pattern, replacement);
-  }
-  return result;
-}
+// Stable placeholder for the built-in tool enumeration the runtime emits when a
+// nonexistent tool is called (see normalizeAvailableToolNames).
+export const availableToolsPlaceholder = "${available_tools}";
 
 // When a model calls a tool that doesn't exist (e.g., the removed report_intent
 // tool), the runtime replies with "Available tools that can be called are <list>."
-// The shell tool family names in that list are platform-specific, so normalize
-// them to placeholders to keep snapshots matching across Windows/Linux/macOS.
+// That enumeration is both platform-specific (shell tool family names differ
+// across OSes) and runtime-version-specific (built-in tools such as write_agent
+// are added or removed over time), so any test that trips this path would break
+// whenever the tool set changes. Collapse the whole list to a stable placeholder
+// so snapshots keep matching as the built-in tool set evolves.
 function normalizeAvailableToolNames(result: string): string {
   return result.replace(
-    /(Available tools that can be called are )([^.]*)/g,
-    (_full, prefix: string, list: string) =>
-      prefix + normalizeShellToolFamilyNames(list),
+    /(Available tools that can be called are )[^.]*/g,
+    (_full, prefix: string) => prefix + availableToolsPlaceholder,
   );
 }