feat: add REPL hints for LLMs using FINAL_VAR

rlm/environments/daytona_repl.py

@@ -261,14 +261,24 @@ def FINAL_VAR(variable_name):
     variable_name = variable_name.strip().strip("\\"\\'")
     if variable_name in _locals:
         return str(_locals[variable_name])
-    return f"Error: Variable '{{variable_name}}' not found"
+    available = [k for k in _locals.keys() if not k.startswith("_")]
+    if available:
+        return f"Error: Variable '{{variable_name}}' not found. Available variables: {{available}}. You must create and assign a variable BEFORE calling FINAL_VAR on it."
+    return f"Error: Variable '{{variable_name}}' not found. No variables have been created yet. You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it."
+
+def SHOW_VARS():
+    available = {{k: type(v).__name__ for k, v in _locals.items() if not k.startswith("_")}}
+    if not available:
+        return "No variables created yet. Use ```repl``` blocks to create variables."
+    return f"Available variables: {{available}}"
 
 _globals = {{
     "__builtins__": __builtins__,
     "__name__": "__main__",
     "llm_query": llm_query,
     "llm_query_batched": llm_query_batched,
     "FINAL_VAR": FINAL_VAR,
+    "SHOW_VARS": SHOW_VARS,
 }}
 
 code = base64.b64decode("{code_b64}").decode()

rlm/environments/docker_repl.py

@@ -142,9 +142,20 @@ def save_state(s):
 
 def FINAL_VAR(name):
     name = name.strip().strip("\\"\\'")
-    return str(_locals.get(name, f"Error: Variable '{{name}}' not found"))
-
-_globals = {{"__builtins__": __builtins__, "__name__": "__main__", "llm_query": llm_query, "llm_query_batched": llm_query_batched, "FINAL_VAR": FINAL_VAR}}
+    if name in _locals:
+        return str(_locals[name])
+    available = [k for k in _locals.keys() if not k.startswith("_")]
+    if available:
+        return f"Error: Variable '{{name}}' not found. Available variables: {{available}}. You must create and assign a variable BEFORE calling FINAL_VAR on it."
+    return f"Error: Variable '{{name}}' not found. No variables have been created yet. You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it."
+
+def SHOW_VARS():
+    available = {{k: type(v).__name__ for k, v in _locals.items() if not k.startswith("_")}}
+    if not available:
+        return "No variables created yet. Use ```repl``` blocks to create variables."
+    return f"Available variables: {{available}}"
+
+_globals = {{"__builtins__": __builtins__, "__name__": "__main__", "llm_query": llm_query, "llm_query_batched": llm_query_batched, "FINAL_VAR": FINAL_VAR, "SHOW_VARS": SHOW_VARS}}
 
 code = base64.b64decode("{code_b64}").decode()
 stdout_buf, stderr_buf = io.StringIO(), io.StringIO()

rlm/environments/local_repl.py

@@ -161,6 +161,7 @@ def setup(self):
 
         # Add helper functions
         self.globals["FINAL_VAR"] = self._final_var
+        self.globals["SHOW_VARS"] = self._show_vars
         self.globals["llm_query"] = self._llm_query
         self.globals["llm_query_batched"] = self._llm_query_batched
 
@@ -169,7 +170,27 @@ def _final_var(self, variable_name: str) -> str:
         variable_name = variable_name.strip().strip("\"'")
         if variable_name in self.locals:
             return str(self.locals[variable_name])
-        return f"Error: Variable '{variable_name}' not found"
+
+        # Provide helpful error message with available variables
+        available = [k for k in self.locals.keys() if not k.startswith("_")]
+        if available:
+            return (
+                f"Error: Variable '{variable_name}' not found. "
+                f"Available variables: {available}. "
+                f"You must create and assign a variable BEFORE calling FINAL_VAR on it."
+            )
+        return (
+            f"Error: Variable '{variable_name}' not found. "
+            f"No variables have been created yet. "
+            f"You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it."
+        )
+
+    def _show_vars(self) -> str:
+        """Show all available variables in the REPL environment."""
+        available = {k: type(v).__name__ for k, v in self.locals.items() if not k.startswith("_")}
+        if not available:
+            return "No variables created yet. Use ```repl``` blocks to create variables."
+        return f"Available variables: {available}"
 
     def _llm_query(self, prompt: str, model: str | None = None) -> str:
         """Query the LM via socket connection to the handler.

rlm/environments/modal_repl.py

@@ -221,14 +221,24 @@ def FINAL_VAR(variable_name):
     variable_name = variable_name.strip().strip("\\"\\'")
     if variable_name in _locals:
         return str(_locals[variable_name])
-    return f"Error: Variable '{{variable_name}}' not found"
+    available = [k for k in _locals.keys() if not k.startswith("_")]
+    if available:
+        return f"Error: Variable '{{variable_name}}' not found. Available variables: {{available}}. You must create and assign a variable BEFORE calling FINAL_VAR on it."
+    return f"Error: Variable '{{variable_name}}' not found. No variables have been created yet. You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it."
+
+def SHOW_VARS():
+    available = {{k: type(v).__name__ for k, v in _locals.items() if not k.startswith("_")}}
+    if not available:
+        return "No variables created yet. Use ```repl``` blocks to create variables."
+    return f"Available variables: {{available}}"
 
 _globals = {{
     "__builtins__": __builtins__,
     "__name__": "__main__",
     "llm_query": llm_query,
     "llm_query_batched": llm_query_batched,
     "FINAL_VAR": FINAL_VAR,
+    "SHOW_VARS": SHOW_VARS,
 }}
 
 code = base64.b64decode("{code_b64}").decode()

rlm/environments/prime_repl.py

@@ -220,14 +220,24 @@ def FINAL_VAR(variable_name):
     variable_name = variable_name.strip().strip("\\"\\'")
     if variable_name in _locals:
         return str(_locals[variable_name])
-    return f"Error: Variable '{{variable_name}}' not found"
+    available = [k for k in _locals.keys() if not k.startswith("_")]
+    if available:
+        return f"Error: Variable '{{variable_name}}' not found. Available variables: {{available}}. You must create and assign a variable BEFORE calling FINAL_VAR on it."
+    return f"Error: Variable '{{variable_name}}' not found. No variables have been created yet. You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it."
+
+def SHOW_VARS():
+    available = {{k: type(v).__name__ for k, v in _locals.items() if not k.startswith("_")}}
+    if not available:
+        return "No variables created yet. Use ```repl``` blocks to create variables."
+    return f"Available variables: {{available}}"
 
 _globals = {{
     "__builtins__": __builtins__,
     "__name__": "__main__",
     "llm_query": llm_query,
     "llm_query_batched": llm_query_batched,
     "FINAL_VAR": FINAL_VAR,
+    "SHOW_VARS": SHOW_VARS,
 }}
 
 code = base64.b64decode("{code_b64}").decode()

rlm/utils/prompts.py

@@ -10,7 +10,8 @@
 1. A `context` variable that contains extremely important information about your query. You should check the content of the `context` variable to understand what you are working with. Make sure you look through it sufficiently as you answer your query.
 2. A `llm_query` function that allows you to query an LLM (that can handle around 500K chars) inside your REPL environment.
 3. A `llm_query_batched` function that allows you to query multiple prompts concurrently: `llm_query_batched(prompts: List[str]) -> List[str]`. This is much faster than sequential `llm_query` calls when you have multiple independent queries. Results are returned in the same order as the input prompts.
-4. The ability to use `print()` statements to view the output of your REPL code and continue your reasoning.
+4. A `SHOW_VARS()` function that returns all variables you have created in the REPL. Use this to check what variables exist before using FINAL_VAR.
+5. The ability to use `print()` statements to view the output of your REPL code and continue your reasoning.
 
 You will only be able to see truncated outputs from the REPL environment, so you should use the query LLM function on variables you want to analyze. You will find this function especially useful when you have to analyze the semantics of the context. Use these variables as buffers to build up your final answer.
 Make sure to explicitly look through the entire context in REPL before answering your query. An example strategy is to first look at the context and figure out a chunking strategy, then break up the context into smart chunks, and query an LLM per chunk with a particular question and save the answers to a buffer, then query an LLM with all the buffers to produce your final answer.
@@ -76,6 +77,15 @@
 1. Use FINAL(your final answer here) to provide the answer directly
 2. Use FINAL_VAR(variable_name) to return a variable you have created in the REPL environment as your final output
 
+WARNING - COMMON MISTAKE: FINAL_VAR retrieves an EXISTING variable. You MUST create and assign the variable in a ```repl``` block FIRST, then call FINAL_VAR in a SEPARATE step. For example:
+- WRONG: Calling FINAL_VAR(my_answer) without first creating `my_answer` in a repl block
+- CORRECT: First run ```repl
+my_answer = "the result"
+print(my_answer)
+``` then in the NEXT response call FINAL_VAR(my_answer)
+
+If you're unsure what variables exist, you can call SHOW_VARS() in a repl block to see all available variables.
+
 Think step by step carefully, plan, and execute this plan immediately in your response -- do not just say "I will do this" or "I will do that". Output to the REPL environment and recursive LLMs as much as possible. Remember to explicitly answer the original query in your final answer.
 """
 )