perf: switch Investigator/Critic baseline to Opus 4.7 (#737)

Author: sudsali

src/scripts/issue_bot/bedrock_client.py

@@ -1,4 +1,5 @@
 import logging
+import re
 
 import boto3
 from botocore.config import Config as BotoConfig
@@ -8,6 +9,23 @@
 
 _CIRCUIT_BREAKER_THRESHOLD = 3
 
+# Models that reject `temperature` (and `top_p`, `top_k`) in inferenceConfig.
+# Opus 4.7 was the first; pattern is anchored at a non-digit boundary so we
+# match `claude-opus-4-7` and `claude-opus-4-7-mini-...` but not a hypothetical
+# `opus-4-70`. Add new families here as they drop sampling params.
+_NO_SAMPLING_PARAMS_PATTERN = re.compile(r"opus-4-7(?!\d)")
+
+
+def _build_inference_config(max_tokens, temperature, model_id):
+    """Compose inferenceConfig for converse(). Drops temperature for models
+    that reject it; leaves the field on older Claude models so we keep the
+    existing 0.3 sampling behavior for Reporter/Haiku/legacy."""
+    cfg = {"maxTokens": max_tokens}
+    if model_id and _NO_SAMPLING_PARAMS_PATTERN.search(model_id):
+        return cfg
+    cfg["temperature"] = temperature
+    return cfg
+
 
 class BedrockClient:
     def __init__(self, cfg):
@@ -82,7 +100,7 @@ def invoke(self, system_prompt, user_prompt, max_tokens=4096,
             kwargs = {
                 "modelId": self._model_id,
                 "messages": [{"role": "user", "content": user_content}],
-                "inferenceConfig": {"maxTokens": max_tokens, "temperature": temperature},
+                "inferenceConfig": _build_inference_config(max_tokens, temperature, self._model_id),
             }
 
             if system_prompt:
@@ -159,10 +177,11 @@ def invoke_with_usage(self, system_prompt, user_prompt, max_tokens=4096,
                 user_content = [{"guardContent": {"text": {"text": user_prompt}}}]
             else:
                 user_content = [{"text": user_prompt}]
+            effective_model = model_id or self._model_id
             kwargs = {
-                "modelId": model_id or self._model_id,
+                "modelId": effective_model,
                 "messages": [{"role": "user", "content": user_content}],
-                "inferenceConfig": {"maxTokens": max_tokens, "temperature": temperature},
+                "inferenceConfig": _build_inference_config(max_tokens, temperature, effective_model),
             }
             if system_prompt:
                 kwargs["system"] = [{"text": system_prompt}]
@@ -224,20 +243,21 @@ def converse_with_tools(self, system_prompt, messages, tool_specs,
         Two cachePoints: one after the system block, one at the tail of the
         last message (so the growing conversation history is cached
         turn-over-turn instead of re-priced as fresh input every turn).
-        Both Opus 4.6 and Haiku 4.5 require ≥4,096 tokens before a cachePoint
-        for it to take effect; first-turn message-tail caches silently no-op
-        on small payloads but become useful once tool results accumulate.
+        Opus 4.x and Haiku 4.5 require ≥4,096 tokens before a cachePoint for
+        it to take effect; first-turn message-tail caches silently no-op on
+        small payloads but become useful once tool results accumulate.
         """
         if self._circuit_open:
             logger.warning("Circuit breaker open, skipping Bedrock tool-use call")
             return None
         try:
             wrapped_messages = self._wrap_first_user_for_guardrail(messages)
             wrapped_messages = self._append_tail_cache_point(wrapped_messages)
+            effective_model = model_id or self._model_id
             kwargs = {
-                "modelId": model_id or self._model_id,
+                "modelId": effective_model,
                 "messages": wrapped_messages,
-                "inferenceConfig": {"maxTokens": max_tokens, "temperature": temperature},
+                "inferenceConfig": _build_inference_config(max_tokens, temperature, effective_model),
             }
             if tool_specs:
                 kwargs["toolConfig"] = {"tools": tool_specs}

src/scripts/issue_bot/config.py

@@ -20,7 +20,7 @@ def __init__(self):
         self.event_before = os.getenv("EVENT_BEFORE", "")
         self.event_after = os.getenv("EVENT_AFTER", "")
 
-        self.bedrock_model_id = os.getenv("BEDROCK_MODEL_ID", "us.anthropic.claude-opus-4-6-v1")
+        self.bedrock_model_id = os.getenv("BEDROCK_MODEL_ID", "us.anthropic.claude-opus-4-7")
         # Reporter is a JSON-formatting + filter step; it doesn't reason. A
         # cheaper model (Haiku 4.5) handles structured output cleanly at ~5x
         # less cost. Defaults to bedrock_model_id so the swap is opt-in via

src/scripts/tests/test_bot.py

@@ -710,6 +710,62 @@ def test_invoke_default_no_cache_prefix(self):
         for block in kwargs.get("system", []):
             assert "cachePoint" not in block
 
+    def test_inference_config_drops_temperature_for_opus_4_7(self):
+        """Opus 4.7 returns 400 if temperature is in inferenceConfig. The
+        helper omits it for any model_id matching the opus-4-7 marker
+        (regional 'us.' or 'global.' inference profiles)."""
+        from issue_bot.bedrock_client import _build_inference_config
+        cfg = _build_inference_config(8000, 0.3, "us.anthropic.claude-opus-4-7")
+        assert "temperature" not in cfg
+        assert cfg["maxTokens"] == 8000
+
+        cfg = _build_inference_config(8000, 0.3, "global.anthropic.claude-opus-4-7")
+        assert "temperature" not in cfg
+
+    def test_inference_config_keeps_temperature_for_other_models(self):
+        from issue_bot.bedrock_client import _build_inference_config
+        cfg = _build_inference_config(4096, 0.5, "us.anthropic.claude-opus-4-6-v1")
+        assert cfg["temperature"] == 0.5
+
+        cfg = _build_inference_config(4096, 0.5, "us.anthropic.claude-haiku-4-5-20251001-v1:0")
+        assert cfg["temperature"] == 0.5
+
+    def test_inference_config_handles_missing_model_id(self):
+        """If model_id is None or empty, fall back to including temperature
+        (preserves legacy behavior — caller is responsible for compatibility)."""
+        from issue_bot.bedrock_client import _build_inference_config
+        cfg = _build_inference_config(4096, 0.3, None)
+        assert cfg["temperature"] == 0.3
+        cfg = _build_inference_config(4096, 0.3, "")
+        assert cfg["temperature"] == 0.3
+
+    def test_converse_with_tools_drops_temperature_end_to_end_for_opus_4_7(self):
+        """End-to-end guard: a future refactor that bypasses _build_inference_config
+        at one of the call sites would slip a 400 ValidationException into
+        production. Assert the wire payload to Bedrock has no temperature when
+        the effective model is Opus 4.7."""
+        client = self._make_client()
+        self._mock_converse(client)
+        client.converse_with_tools(
+            "system",
+            [{"role": "user", "content": [{"text": "x"}]}],
+            tool_specs=[],
+            model_id="us.anthropic.claude-opus-4-7",
+        )
+        kwargs = client._client.converse.call_args[1]
+        assert "temperature" not in kwargs["inferenceConfig"]
+        assert kwargs["inferenceConfig"]["maxTokens"] > 0
+
+    def test_invoke_drops_temperature_for_opus_4_7_default(self):
+        """invoke() takes no model_id arg; uses self._model_id. When Config
+        defaults to Opus 4.7, the wire payload must drop temperature."""
+        client = self._make_client()
+        client._model_id = "us.anthropic.claude-opus-4-7"
+        self._mock_converse(client)
+        client.invoke("system", "user")
+        kwargs = client._client.converse.call_args[1]
+        assert "temperature" not in kwargs["inferenceConfig"]
+
     def test_converse_with_tools_total_cache_points_under_limit(self):
         """Bedrock rejects requests with more than 4 cachePoints. Sum across
         system + every message content block to catch any future addition."""