fix(lisp): prevent closure and HOF state leaks

Stop closures from retaining turn history and resolve *1/*2/*3 from the active caller context. Clean up the higher-order-function side-effect stash on error paths, and avoid registry atom load-order failures for static builtin names.

Verified with mix test.

Author: andreasronge

lib/ptc_runner/lisp/eval.ex

@@ -402,14 +402,14 @@ defmodule PtcRunner.Lisp.Eval do
     # would inflate session memory by ~18 KB per closure.
     {captured_env, captured_locals} = capture_lexical_scope(eval_ctx)
     meta = locals_meta(captured_locals, %{})
-    {:ok, {:closure, params, body, captured_env, eval_ctx.turn_history, meta}, eval_ctx}
+    {:ok, {:closure, params, body, captured_env, [], meta}, eval_ctx}
   end
 
   # Named fn: (fn name [params] body) — name is bound inside body for self-recursion
   defp do_eval({:fn, name, params, body}, %EvalContext{} = eval_ctx) do
     {captured_env, captured_locals} = capture_lexical_scope(eval_ctx)
     meta = locals_meta(captured_locals, %{fn_name: name})
-    {:ok, {:closure, params, body, captured_env, eval_ctx.turn_history, meta}, eval_ctx}
+    {:ok, {:closure, params, body, captured_env, [], meta}, eval_ctx}
   end
 
   # ============================================================
@@ -1235,7 +1235,7 @@ defmodule PtcRunner.Lisp.Eval do
 
   # Convert a closure to a zero-arity Erlang function for use in pcalls
   defp pcalls_fn_to_erlang(
-         {:closure, [], body, closure_env, turn_history, metadata} = closure,
+         {:closure, [], body, closure_env, _turn_history, metadata} = closure,
          %EvalContext{} = eval_ctx
        ) do
     closure_env = Apply.bind_self_recursion(closure_env, metadata, closure)
@@ -1247,7 +1247,7 @@ defmodule PtcRunner.Lisp.Eval do
           eval_ctx.user_ns,
           closure_env,
           eval_ctx.tool_exec,
-          turn_history
+          eval_ctx.turn_history
         )
 
       ctx = %{

lib/ptc_runner/lisp/eval/apply.ex

@@ -127,9 +127,7 @@ defmodule PtcRunner.Lisp.Eval.Apply do
     converted_args = Enum.map(args, fn arg -> closure_to_fun(arg, eval_ctx, do_eval_fn) end)
 
     try do
-      push_side_effect_stash()
-      result = apply(fun, converted_args)
-      {:ok, result, pop_side_effects(eval_ctx)}
+      with_side_effect_stash(eval_ctx, fn -> apply(fun, converted_args) end)
     rescue
       FunctionClauseError ->
         # Provide a helpful error message for type mismatches
@@ -261,12 +259,10 @@ defmodule PtcRunner.Lisp.Eval.Apply do
        when is_function(fun, 1) do
     # Convert any closures/builtins in args to callable functions
     converted_args = Enum.map(args, fn arg -> closure_to_fun(arg, eval_ctx, do_eval_fn) end)
-    push_side_effect_stash()
-    result = fun.(converted_args)
-    {:ok, result, pop_side_effects(eval_ctx)}
+
+    with_side_effect_stash(eval_ctx, fn -> fun.(converted_args) end)
   rescue
     e in ExecutionError ->
-      pop_side_effects(eval_ctx)
       {:error, execution_error_tuple(e)}
   end
 
@@ -286,9 +282,7 @@ defmodule PtcRunner.Lisp.Eval.Apply do
       fun = elem(funs, idx)
 
       try do
-        push_side_effect_stash()
-        result = apply(fun, converted_args)
-        {:ok, result, pop_side_effects(eval_ctx)}
+        with_side_effect_stash(eval_ctx, fn -> apply(fun, converted_args) end)
       rescue
         FunctionClauseError ->
           # Provide a helpful error message for type mismatches
@@ -308,9 +302,7 @@ defmodule PtcRunner.Lisp.Eval.Apply do
 
   defp do_apply_fun(fun, args, %EvalContext{} = eval_ctx, _do_eval_fn)
        when is_function(fun) do
-    push_side_effect_stash()
-    result = apply(fun, args)
-    {:ok, result, pop_side_effects(eval_ctx)}
+    with_side_effect_stash(eval_ctx, fn -> apply(fun, args) end)
   end
 
   # Map as function: (map key) → Map.get(map, key)
@@ -598,7 +590,7 @@ defmodule PtcRunner.Lisp.Eval.Apply do
          body,
          closure_env,
          %EvalContext{} = eval_context,
-         closure_turn_history,
+         _closure_turn_history,
          metadata,
          do_eval_fn
        ) do
@@ -628,7 +620,7 @@ defmodule PtcRunner.Lisp.Eval.Apply do
         eval_context.user_ns,
         new_env,
         eval_context.tool_exec,
-        closure_turn_history,
+        eval_context.turn_history,
         pmap_timeout: eval_context.pmap_timeout,
         catalog_exec: eval_context.catalog_exec
       )
@@ -656,11 +648,35 @@ defmodule PtcRunner.Lisp.Eval.Apply do
   # Uses a stack to handle nested HOFs: each HOF pushes a fresh accumulator,
   # inner HOFs push/pop their own level, and the outer HOF collects everything.
 
+  defp with_side_effect_stash(%EvalContext{} = eval_ctx, fun) when is_function(fun, 0) do
+    push_side_effect_stash()
+
+    try do
+      result = fun.()
+      {:ok, result, pop_side_effects(eval_ctx)}
+    rescue
+      e ->
+        drop_side_effect_stash()
+        reraise e, __STACKTRACE__
+    catch
+      kind, reason ->
+        drop_side_effect_stash()
+        :erlang.raise(kind, reason, __STACKTRACE__)
+    end
+  end
+
   defp push_side_effect_stash do
     stack = Process.get(:__ptc_hof_stack, [])
     Process.put(:__ptc_hof_stack, [%{tool_calls: [], prints: [], catalog_ops: []} | stack])
   end
 
+  defp drop_side_effect_stash do
+    case Process.get(:__ptc_hof_stack, []) do
+      [_top | rest] -> Process.put(:__ptc_hof_stack, rest)
+      [] -> :ok
+    end
+  end
+
   defp stash_side_effects(%EvalContext{} = ctx) do
     case Process.get(:__ptc_hof_stack, []) do
       [top | rest] ->
@@ -737,7 +753,7 @@ defmodule PtcRunner.Lisp.Eval.Apply do
   end
 
   defp do_execute_closure(
-         {:closure, _closure_patterns, body, closure_env, closure_turn_history, meta} = closure,
+         {:closure, _closure_patterns, body, closure_env, _closure_turn_history, meta} = closure,
          binding_patterns,
          args,
          %EvalContext{ctx: ctx, user_ns: user_ns, tool_exec: tool_exec} = caller_ctx,
@@ -754,7 +770,7 @@ defmodule PtcRunner.Lisp.Eval.Apply do
             _ -> new_env
           end
 
-        closure_ctx = EvalContext.new(ctx, user_ns, new_env, tool_exec, closure_turn_history)
+        closure_ctx = EvalContext.new(ctx, user_ns, new_env, tool_exec, caller_ctx.turn_history)
 
         # Carry accumulated state from caller so tool_calls/cache aren't lost across closure calls.
         # `locals` is rebuilt from the closure's lexical capture + this invocation's
@@ -844,7 +860,7 @@ defmodule PtcRunner.Lisp.Eval.Apply do
       Enum.reduce(user_ns, user_ns, fn
         {name, {:closure, ^params, ^body, ^env, ^th, old_meta}}, acc ->
           new_meta = Map.put(old_meta, :return_type, return_type)
-          Map.put(acc, name, {:closure, params, body, env, th, new_meta})
+          Map.put(acc, name, {:closure, params, body, env, [], new_meta})
 
         _, acc ->
           acc

lib/ptc_runner/lisp/registry.ex

@@ -123,7 +123,9 @@ defmodule PtcRunner.Lisp.Registry do
   def builtins_by_category(category) do
     implemented()
     |> Enum.filter(&(&1.category == category and &1.dispatch == :env))
-    |> Enum.map(&String.to_existing_atom(&1.name))
+    # Names come from the closed compile-time registry, not user input.
+    # Using to_atom/1 avoids depending on Env being loaded first.
+    |> Enum.map(&String.to_atom(&1.name))
   end
 
   @doc """

test/ptc_runner/lisp/closure_capture_test.exs

@@ -195,5 +195,43 @@ defmodule PtcRunner.Lisp.ClosureCaptureTest do
       assert memory_size < 2_000,
              "expected defn memory < 2000 bytes, got #{memory_size}"
     end
+
+    test "closures do not retain prior turn history in session memory" do
+      large_history_entry = String.duplicate("x", 100_000)
+
+      {:ok, step} =
+        Lisp.run("(def f (fn [] 1))",
+          profile: :mcp_no_tools,
+          mode: :multi_turn,
+          turn_history: [large_history_entry]
+        )
+
+      memory_size = :erlang.external_size(step.memory)
+      {:closure, _params, _body, _env, captured_history, _meta} = step.memory[:f]
+
+      assert captured_history == []
+
+      assert memory_size < 2_000,
+             "expected closure memory to exclude turn history, got #{memory_size} bytes"
+    end
+
+    test "closure reads current caller turn history, not definition-time history" do
+      {:ok, step1} =
+        Lisp.run("(def f (fn [] *1))",
+          profile: :mcp_no_tools,
+          mode: :multi_turn,
+          turn_history: ["definition-time"]
+        )
+
+      {:ok, step2} =
+        Lisp.run("(f)",
+          profile: :mcp_no_tools,
+          mode: :multi_turn,
+          memory: step1.memory,
+          turn_history: ["call-time"]
+        )
+
+      assert step2.return == "call-time"
+    end
   end
 end

test/ptc_runner/lisp/hof_side_effects_test.exs

@@ -11,6 +11,16 @@ defmodule PtcRunner.Lisp.HofSideEffectsTest do
   use ExUnit.Case, async: true
 
   alias PtcRunner.Lisp
+  alias PtcRunner.Lisp.{Env, Eval}
+
+  defp clear_hof_stack! do
+    Process.delete(:__ptc_hof_stack)
+    on_exit(fn -> Process.delete(:__ptc_hof_stack) end)
+  end
+
+  defp assert_hof_stack_empty do
+    assert Process.get(:__ptc_hof_stack, []) == []
+  end
 
   describe "tool_calls inside reduce" do
     test "tool calls made inside reduce are captured in step.tool_calls" do
@@ -140,4 +150,60 @@ defmodule PtcRunner.Lisp.HofSideEffectsTest do
       assert step.prints == ["storing 1", "storing 2", "storing 3"]
     end
   end
+
+  describe "side-effect stash cleanup on errors" do
+    test "multi-arity HOF callback errors do not retain process dictionary state" do
+      clear_hof_stack!()
+
+      ast =
+        {:call, {:var, :map},
+         [
+           {:fn, [{:var, :x}], {:call, {:var, :+}, [{:var, :x}, nil]}},
+           {:vector, [1, 2, 3]}
+         ]}
+
+      assert {:error, {:type_error, _message, _args}} =
+               Eval.eval(ast, %{}, %{}, Env.initial(), fn _, _ -> nil end)
+
+      assert_hof_stack_empty()
+    end
+
+    test "normal HOF callback errors do not retain process dictionary state" do
+      clear_hof_stack!()
+
+      ast =
+        {:call, {:var, :filter},
+         [
+           {:fn, [{:var, :x}], {:call, {:var, :+}, [{:var, :x}, nil]}},
+           {:vector, [1, 2, 3]}
+         ]}
+
+      assert {:error, {:type_error, _message, _args}} =
+               Eval.eval(ast, %{}, %{}, Env.initial(), fn _, _ -> nil end)
+
+      assert_hof_stack_empty()
+    end
+
+    test "collect builtin errors do not retain process dictionary state" do
+      clear_hof_stack!()
+
+      assert_raise ArgumentError, ~r/every-pred requires at least 1 predicate/, fn ->
+        Eval.eval({:call, {:var, :"every-pred"}, []}, %{}, %{}, Env.initial(), fn _, _ -> nil end)
+      end
+
+      assert_hof_stack_empty()
+    end
+
+    test "plain function errors do not retain process dictionary state" do
+      clear_hof_stack!()
+
+      bad = fn _ -> raise "boom" end
+
+      assert_raise RuntimeError, "boom", fn ->
+        Eval.eval({:call, {:var, :bad}, [1]}, %{}, %{}, %{bad: bad}, fn _, _ -> nil end)
+      end
+
+      assert_hof_stack_empty()
+    end
+  end
 end