Skip to content

Commit 9dd1252

Browse files
committed
Replace runtime-type quotation with splice-into-source typechecking
Type-check LLM-generated Callables by splicing them into the Template's real module source -- in place at the template def's position -- and running mypy on the whole module, raising only on diagnostics inside the spliced region. This removes the brittle quotation layer (type_to_ast, signature_to_ast, collect_*, _RenameTransformer) and the runtime/local-type stubbing it depended on, which mishandled common types such as Enum and runtime-created Pydantic models. - mypy_type_check now takes (generated, anchor); the anchor (Template.__default__) is provided by the type_check_anchor effect op, bound per call by Template.__apply__ (so nested synthesis sees the right one) and rebound to None around tool-argument decoding, so the argument path skips the check. - Region-scoped diagnostics: unrelated pre-existing errors elsewhere in the module never block synthesis (no spurious retry burn). - Leaves the template def's decorators untouched (mypy checks the body against its declared return type regardless of decorators); explicit pre-scan for non-nestable constructs (star-import, __future__); source recovery via getsourcelines/file/ linecache (REPL-defined templates); fatal mypy errors are surfaced not swallowed. - Name collisions (#542) are handled structurally by nesting/shadowing, so the AST-rename pass is deleted. - Delete the quotation emitters + their unit tests; add behavior tests covering collision shadowing, the gate, closures, method/static/class templates, Enum/dataclass context, injected-name, non-nestable, and linecache recovery.
1 parent 72bb624 commit 9dd1252

5 files changed

Lines changed: 523 additions & 2000 deletions

File tree

effectful/handlers/llm/completions.py

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
Encodable,
3131
to_content_blocks,
3232
)
33-
from effectful.handlers.llm.evaluation import ReplSession
33+
from effectful.handlers.llm.evaluation import ReplSession, type_check_anchor
3434
from effectful.handlers.llm.template import (
3535
Agent,
3636
Template,
@@ -430,15 +430,18 @@ def call_assistant[T](
430430
encoding: pydantic.TypeAdapter[DecodedToolCall] = pydantic.TypeAdapter(
431431
Encodable[DecodedToolCall]
432432
)
433-
for raw_tool_call in message.get("tool_calls") or []:
434-
try:
435-
tool_calls += [encoding.validate_python(raw_tool_call, context=tools)]
436-
except Exception as e:
437-
raise ToolCallDecodingError(
438-
raw_tool_call=raw_tool_call,
439-
original_error=e,
440-
raw_message=raw_message,
441-
) from e
433+
# Tool-argument Callables have no source anchor (out of scope for the splice
434+
# type check), so suppress the anchor while decoding them.
435+
with handler({type_check_anchor: lambda: None}):
436+
for raw_tool_call in message.get("tool_calls") or []:
437+
try:
438+
tool_calls += [encoding.validate_python(raw_tool_call, context=tools)]
439+
except Exception as e:
440+
raise ToolCallDecodingError(
441+
raw_tool_call=raw_tool_call,
442+
original_error=e,
443+
raw_message=raw_message,
444+
) from e
442445

443446
result = None
444447
if not tool_calls:
@@ -672,7 +675,15 @@ def _call[**P, T](
672675
) # type: ignore
673676
history_copy = history.copy()
674677

675-
with handler({_get_history: lambda: history_copy}):
678+
# Bind the type-check anchor to this Template's underlying function for the
679+
# duration of the call. Handler scope (not data context) means nested or
680+
# recursive synthesis sees the right anchor -- innermost wins.
681+
with handler(
682+
{
683+
_get_history: lambda: history_copy,
684+
type_check_anchor: lambda: template.__default__,
685+
}
686+
):
676687
if (
677688
not _get_history()
678689
or next(iter(_get_history().values()))["role"] != "system"

effectful/handlers/llm/encoding.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -551,7 +551,13 @@ def _validate(value: Any, info: pydantic.ValidationInfo) -> Callable:
551551
)
552552

553553
_validate_signature_ast(last_stmt, expected_params)
554-
evaluation.type_check(module, ctx, expected_params, expected_return)
554+
555+
# `type_check_anchor` is the Template's underlying function, bound per call
556+
# by Template.__apply__; it is None for tool-argument decoding, which
557+
# therefore skips the source-anchored type check.
558+
anchor = evaluation.type_check_anchor()
559+
if anchor is not None:
560+
evaluation.type_check(module, anchor)
555561

556562
g: MutableMapping[str, Any] = {}
557563
g.update(ctx)

0 commit comments

Comments
 (0)