Gate synthetic lexical-reader generation behind LexicalReaders handler

Per eb8680: needing to coach the LLM around noisy lexical-reader tools
in `test_handlers_llm_tool_calling_poem.py` was a smell.  Make the
generation opt-in instead.

- New `expose_lexical_readers()` Operation in `completions.py`, default
  return `False`.  `_collect_tools` gates the reader branch on it.
- New `LexicalReaders` ObjectInterpretation overrides the Operation to
  return `True`; users install it for the call sites where the LLM
  should see closure state.
- Revert the poem prompt-hack: the docstring no longer has to ask the
  LLM to ignore read-only lexical readers, because they are now off
  by default.
- Test A (`test_template_synthesis_uses_lexical_reader`) and the
  reader-integration test (`test_llm_reads_lexical_value`) install
  `LexicalReaders` in their handler stack.  Both fixtures re-recorded
  against gpt-4o-mini.
- Template tests that exercise reader generation install the handler.
  New `test_lexical_readers_off_by_default` and
  `test_lexical_readers_handler_enables_collection` pin both sides of
  the gate.
- `test_template_method` / `test_template_method_nested_class`:
  drop the side-note `"local_variable" in tools` assertions; those
  pinned implicit reader generation.  The core method-template tool
  collection (random, reverse, etc.) is the actual point and still
  passes.
- `test_template_exposes_lexical_classes` now pins both off (default)
  and on (handler installed) — the #497 motivating example with
  explicit gate semantics.

Author: datvo06

effectful/handlers/llm/completions.py

@@ -219,12 +219,40 @@ def tool_fn():
         return super().define(tool_fn)
 
 
+@Operation.define
+def expose_lexical_readers() -> bool:
+    """Effect controlling whether `_collect_tools` builds synthetic
+    read-only Tools for non-Tool/Template values in a Template's
+    lexical scope.
+
+    Default behaviour is *off*: only real Tools/Templates/Agents reach
+    the LLM, and the lexical context is invisible.  Install
+    `LexicalReaders` to flip it on for the call-site where the LLM
+    should be able to inspect closure state.
+    """
+    return False
+
+
+class LexicalReaders(ObjectInterpretation):
+    """Handler that enables synthetic lexical-reader generation in
+    `_collect_tools`.  Each plain value in a Template's lexical context
+    becomes a zero-argument Tool that returns the captured value.
+    """
+
+    @implements(expose_lexical_readers)
+    def _enabled(self) -> bool:
+        return True
+
+
 def _collect_tools(
     env: collections.abc.Mapping[str, typing.Any],
 ) -> collections.abc.Mapping[str, Tool]:
-    """Operations and Templates available as tools, plus synthetic
-    readers for other lexical symbols.  Auto-captured from lexical context."""
+    """Operations and Templates available as tools.  When
+    `expose_lexical_readers` is on (see :class:`LexicalReaders`),
+    plain values in the lexical context are also wrapped as synthetic
+    read-only tools."""
     result: dict[str, Tool] = {}
+    readers_on = expose_lexical_readers()
 
     for name, obj in env.items():
         if isinstance(obj, Tool | Template):
@@ -234,7 +262,7 @@ def _collect_tools(
                 for attr_name in vars(cls):
                     if isinstance(getattr(obj, attr_name), Tool):
                         result[f"{name}__{attr_name}"] = getattr(obj, attr_name)
-        elif name.isidentifier():
+        elif readers_on and name.isidentifier():
             try:
                 result[name] = _LexicalVariableTool.define(obj, name=name)
             # `TypeError` joins the three Pydantic errors because the

tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value.json

@@ -1,9 +1,9 @@
 {
-  "id": "chatcmpl-Dl3k0fEKWeTJpZDy3rcqAKPKE46Cu",
-  "created": 1780108168,
+  "id": "chatcmpl-DoaWCNbxu6WP3Lx3l3Dmkmr5CBxwg",
+  "created": 1780949148,
   "model": "gpt-4o-mini-2024-07-18",
   "object": "chat.completion",
-  "system_fingerprint": "fp_df8c8d3b43",
+  "system_fingerprint": "fp_6c2953e649",
   "choices": [
     {
       "finish_reason": "tool_calls",
@@ -17,7 +17,7 @@
               "arguments": "{}",
               "name": "_known_data"
             },
-            "id": "call_DoVBVCnKiU5k7FTzWRo5nnzm",
+            "id": "call_nEkf11zJbFZRFcBclPCAvTao",
             "type": "function"
           }
         ],
@@ -32,8 +32,8 @@
   ],
   "usage": {
     "completion_tokens": 11,
-    "prompt_tokens": 6043,
-    "total_tokens": 6054,
+    "prompt_tokens": 3753,
+    "total_tokens": 3764,
     "completion_tokens_details": {
       "accepted_prediction_tokens": 0,
       "audio_tokens": 0,
@@ -45,7 +45,7 @@
     },
     "prompt_tokens_details": {
       "audio_tokens": 0,
-      "cached_tokens": 0,
+      "cached_tokens": 2816,
       "text_tokens": null,
       "image_tokens": null,
       "video_tokens": null

tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_1.json

@@ -1,9 +1,9 @@
 {
-  "id": "chatcmpl-Dl3k2MUxG3A1S3HrmgwMXFmDNScCt",
-  "created": 1780108170,
+  "id": "chatcmpl-DoaWFqScxo1oUrffPxQIyJYv8ENNr",
+  "created": 1780949151,
   "model": "gpt-4o-mini-2024-07-18",
   "object": "chat.completion",
-  "system_fingerprint": "fp_df8c8d3b43",
+  "system_fingerprint": "fp_6c2953e649",
   "choices": [
     {
       "finish_reason": "tool_calls",
@@ -17,7 +17,7 @@
               "arguments": "{\"a\":10,\"b\":20}",
               "name": "add_numbers"
             },
-            "id": "call_vypFiLTjH5MEpMKSL3VLoD4z",
+            "id": "call_TABbctvXDmYyjourZxIgLFo1",
             "type": "function"
           }
         ],
@@ -32,8 +32,8 @@
   ],
   "usage": {
     "completion_tokens": 18,
-    "prompt_tokens": 6077,
-    "total_tokens": 6095,
+    "prompt_tokens": 3787,
+    "total_tokens": 3805,
     "completion_tokens_details": {
       "accepted_prediction_tokens": 0,
       "audio_tokens": 0,
@@ -45,7 +45,7 @@
     },
     "prompt_tokens_details": {
       "audio_tokens": 0,
-      "cached_tokens": 6016,
+      "cached_tokens": 3712,
       "text_tokens": null,
       "image_tokens": null,
       "video_tokens": null

tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_2.json

@@ -1,9 +1,9 @@
 {
-  "id": "chatcmpl-Dl3k5bfkN00lZh8L5D82Cr9YhOJsA",
-  "created": 1780108173,
+  "id": "chatcmpl-DoaWHTP5RK0QyZdDhiHbaUNkv3IfS",
+  "created": 1780949153,
   "model": "gpt-4o-mini-2024-07-18",
   "object": "chat.completion",
-  "system_fingerprint": "fp_df8c8d3b43",
+  "system_fingerprint": "fp_6c2953e649",
   "choices": [
     {
       "finish_reason": "tool_calls",
@@ -17,7 +17,7 @@
               "arguments": "{\"a\":30,\"b\":30}",
               "name": "add_numbers"
             },
-            "id": "call_EmO3tVtqZBqdL6K4dUqLmfgn",
+            "id": "call_SpMwdgqjznHPUWgE19dxgSnc",
             "type": "function"
           }
         ],
@@ -32,8 +32,8 @@
   ],
   "usage": {
     "completion_tokens": 18,
-    "prompt_tokens": 6104,
-    "total_tokens": 6122,
+    "prompt_tokens": 3814,
+    "total_tokens": 3832,
     "completion_tokens_details": {
       "accepted_prediction_tokens": 0,
       "audio_tokens": 0,
@@ -45,7 +45,7 @@
     },
     "prompt_tokens_details": {
       "audio_tokens": 0,
-      "cached_tokens": 6016,
+      "cached_tokens": 3712,
       "text_tokens": null,
       "image_tokens": null,
       "video_tokens": null

tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_3.json

@@ -1,9 +1,9 @@
 {
-  "id": "chatcmpl-Dl3k7gKpQZx4an1oAXAhdYViG1NjR",
-  "created": 1780108175,
+  "id": "chatcmpl-DoaWJQAi5ihtJlJL7algRwDhzcgcx",
+  "created": 1780949155,
   "model": "gpt-4o-mini-2024-07-18",
   "object": "chat.completion",
-  "system_fingerprint": "fp_df8c8d3b43",
+  "system_fingerprint": "fp_6c2953e649",
   "choices": [
     {
       "finish_reason": "tool_calls",
@@ -17,7 +17,7 @@
               "arguments": "{\"a\":60,\"b\":40}",
               "name": "add_numbers"
             },
-            "id": "call_WhTflV4rMdUJUJOY2qqOHv7x",
+            "id": "call_IZ66N2KVq8wyYt3mAOIqfxW0",
             "type": "function"
           }
         ],
@@ -32,8 +32,8 @@
   ],
   "usage": {
     "completion_tokens": 18,
-    "prompt_tokens": 6131,
-    "total_tokens": 6149,
+    "prompt_tokens": 3841,
+    "total_tokens": 3859,
     "completion_tokens_details": {
       "accepted_prediction_tokens": 0,
       "audio_tokens": 0,
@@ -45,7 +45,7 @@
     },
     "prompt_tokens_details": {
       "audio_tokens": 0,
-      "cached_tokens": 6016,
+      "cached_tokens": 3712,
       "text_tokens": null,
       "image_tokens": null,
       "video_tokens": null

tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_4.json

@@ -1,9 +1,9 @@
 {
-  "id": "chatcmpl-Dl3k86CG3iPs5P7110XgQ3Eso2CfZ",
-  "created": 1780108176,
+  "id": "chatcmpl-DoaWLlSqQM5WHmGa4dcjSRypJ0EXu",
+  "created": 1780949157,
   "model": "gpt-4o-mini-2024-07-18",
   "object": "chat.completion",
-  "system_fingerprint": "fp_df8c8d3b43",
+  "system_fingerprint": "fp_6c2953e649",
   "choices": [
     {
       "finish_reason": "tool_calls",
@@ -17,7 +17,7 @@
               "arguments": "{\"a\":100,\"b\":50}",
               "name": "add_numbers"
             },
-            "id": "call_LTy1WHCfWTFnwZ5GSAUElfDA",
+            "id": "call_0unsPGaZPBHhTLObCfc5zAgf",
             "type": "function"
           }
         ],
@@ -32,8 +32,8 @@
   ],
   "usage": {
     "completion_tokens": 18,
-    "prompt_tokens": 6158,
-    "total_tokens": 6176,
+    "prompt_tokens": 3868,
+    "total_tokens": 3886,
     "completion_tokens_details": {
       "accepted_prediction_tokens": 0,
       "audio_tokens": 0,
@@ -45,7 +45,7 @@
     },
     "prompt_tokens_details": {
       "audio_tokens": 0,
-      "cached_tokens": 6016,
+      "cached_tokens": 3840,
       "text_tokens": null,
       "image_tokens": null,
       "video_tokens": null

tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_llm_reads_lexical_value_5.json

@@ -1,9 +1,9 @@
 {
-  "id": "chatcmpl-Dl3k9lFSuyqAQbfM8J1H3BQmgYopC",
-  "created": 1780108177,
+  "id": "chatcmpl-DoaWNFKpBO8KO9IuPrwpMf2aDayBQ",
+  "created": 1780949159,
   "model": "gpt-4o-mini-2024-07-18",
   "object": "chat.completion",
-  "system_fingerprint": "fp_df8c8d3b43",
+  "system_fingerprint": "fp_6c2953e649",
   "choices": [
     {
       "finish_reason": "stop",
@@ -23,8 +23,8 @@
   ],
   "usage": {
     "completion_tokens": 11,
-    "prompt_tokens": 6185,
-    "total_tokens": 6196,
+    "prompt_tokens": 3895,
+    "total_tokens": 3906,
     "completion_tokens_details": {
       "accepted_prediction_tokens": 0,
       "audio_tokens": 0,
@@ -36,7 +36,7 @@
     },
     "prompt_tokens_details": {
       "audio_tokens": 0,
-      "cached_tokens": 6144,
+      "cached_tokens": 3840,
       "text_tokens": null,
       "image_tokens": null,
       "video_tokens": null

tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader.json

@@ -1,6 +1,6 @@
 {
-  "id": "chatcmpl-DoXmRMT0faR6nMFTGqu3wIgzXKSki",
-  "created": 1780938623,
+  "id": "chatcmpl-DoaVmdWxIVHC9IAfb8gM7TohRDtBb",
+  "created": 1780949122,
   "model": "gpt-4o-mini-2024-07-18",
   "object": "chat.completion",
   "system_fingerprint": "fp_6c2953e649",
@@ -17,7 +17,7 @@
               "arguments": "{}",
               "name": "threshold"
             },
-            "id": "call_ypeSxtaJp5QNiymevzksgMSO",
+            "id": "call_LZbU6rs5AtcMphaKxdwuERBs",
             "type": "function"
           }
         ],
@@ -32,8 +32,8 @@
   ],
   "usage": {
     "completion_tokens": 9,
-    "prompt_tokens": 3897,
-    "total_tokens": 3906,
+    "prompt_tokens": 3937,
+    "total_tokens": 3946,
     "completion_tokens_details": {
       "accepted_prediction_tokens": 0,
       "audio_tokens": 0,

tests/fixtures/tests_test_handlers_llm_provider.py__TestSyntheticReaderIntegration__test_template_synthesis_uses_lexical_reader_1.json

@@ -1,6 +1,6 @@
 {
-  "id": "chatcmpl-DoXmTBNxEKjwPzBRAJJUzwj3sKj5r",
-  "created": 1780938625,
+  "id": "chatcmpl-DoaVqPg5WV0Vjsp6Mklax4LPZSAD1",
+  "created": 1780949126,
   "model": "gpt-4o-mini-2024-07-18",
   "object": "chat.completion",
   "system_fingerprint": "fp_6c2953e649",
@@ -23,8 +23,8 @@
   ],
   "usage": {
     "completion_tokens": 31,
-    "prompt_tokens": 3916,
-    "total_tokens": 3947,
+    "prompt_tokens": 3956,
+    "total_tokens": 3987,
     "completion_tokens_details": {
       "accepted_prediction_tokens": 0,
       "audio_tokens": 0,
@@ -36,7 +36,7 @@
     },
     "prompt_tokens_details": {
       "audio_tokens": 0,
-      "cached_tokens": 3840,
+      "cached_tokens": 0,
       "text_tokens": null,
       "image_tokens": null,
       "video_tokens": null