From 9a95a2bb38a73499473bc830b5b1b0806e79ce47 Mon Sep 17 00:00:00 2001 From: Raymond Weitekamp Date: Wed, 28 Jan 2026 07:37:53 -0500 Subject: [PATCH] feat: add REPL hints for LLMs using FINAL_VAR - Improved FINAL_VAR error messages to show available variables and explain that variables must be created before use - Added SHOW_VARS() helper function to all REPL environments - Updated system prompt with WARNING section about common mistake - Documented SHOW_VARS() in REPL initialization instructions This helps weaker LLMs (like gemini-3-flash-preview) avoid the common mistake of calling FINAL_VAR before creating the variable. Co-Authored-By: Claude Opus 4.5 --- rlm/environments/daytona_repl.py | 12 +++++++++++- rlm/environments/docker_repl.py | 17 ++++++++++++++--- rlm/environments/local_repl.py | 23 ++++++++++++++++++++++- rlm/environments/modal_repl.py | 12 +++++++++++- rlm/environments/prime_repl.py | 12 +++++++++++- rlm/utils/prompts.py | 12 +++++++++++- 6 files changed, 80 insertions(+), 8 deletions(-) diff --git a/rlm/environments/daytona_repl.py b/rlm/environments/daytona_repl.py index e76521917..f5cf267a4 100644 --- a/rlm/environments/daytona_repl.py +++ b/rlm/environments/daytona_repl.py @@ -261,7 +261,16 @@ def FINAL_VAR(variable_name): variable_name = variable_name.strip().strip("\\"\\'") if variable_name in _locals: return str(_locals[variable_name]) - return f"Error: Variable '{{variable_name}}' not found" + available = [k for k in _locals.keys() if not k.startswith("_")] + if available: + return f"Error: Variable '{{variable_name}}' not found. Available variables: {{available}}. You must create and assign a variable BEFORE calling FINAL_VAR on it." + return f"Error: Variable '{{variable_name}}' not found. No variables have been created yet. You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it." + +def SHOW_VARS(): + available = {{k: type(v).__name__ for k, v in _locals.items() if not k.startswith("_")}} + if not available: + return "No variables created yet. Use ```repl``` blocks to create variables." + return f"Available variables: {{available}}" _globals = {{ "__builtins__": __builtins__, @@ -269,6 +278,7 @@ def FINAL_VAR(variable_name): "llm_query": llm_query, "llm_query_batched": llm_query_batched, "FINAL_VAR": FINAL_VAR, + "SHOW_VARS": SHOW_VARS, }} code = base64.b64decode("{code_b64}").decode() diff --git a/rlm/environments/docker_repl.py b/rlm/environments/docker_repl.py index e4bd6dff3..2de8c2b26 100644 --- a/rlm/environments/docker_repl.py +++ b/rlm/environments/docker_repl.py @@ -142,9 +142,20 @@ def save_state(s): def FINAL_VAR(name): name = name.strip().strip("\\"\\'") - return str(_locals.get(name, f"Error: Variable '{{name}}' not found")) - -_globals = {{"__builtins__": __builtins__, "__name__": "__main__", "llm_query": llm_query, "llm_query_batched": llm_query_batched, "FINAL_VAR": FINAL_VAR}} + if name in _locals: + return str(_locals[name]) + available = [k for k in _locals.keys() if not k.startswith("_")] + if available: + return f"Error: Variable '{{name}}' not found. Available variables: {{available}}. You must create and assign a variable BEFORE calling FINAL_VAR on it." + return f"Error: Variable '{{name}}' not found. No variables have been created yet. You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it." + +def SHOW_VARS(): + available = {{k: type(v).__name__ for k, v in _locals.items() if not k.startswith("_")}} + if not available: + return "No variables created yet. Use ```repl``` blocks to create variables." + return f"Available variables: {{available}}" + +_globals = {{"__builtins__": __builtins__, "__name__": "__main__", "llm_query": llm_query, "llm_query_batched": llm_query_batched, "FINAL_VAR": FINAL_VAR, "SHOW_VARS": SHOW_VARS}} code = base64.b64decode("{code_b64}").decode() stdout_buf, stderr_buf = io.StringIO(), io.StringIO() diff --git a/rlm/environments/local_repl.py b/rlm/environments/local_repl.py index 5aa1b696c..c37cd2c3b 100644 --- a/rlm/environments/local_repl.py +++ b/rlm/environments/local_repl.py @@ -161,6 +161,7 @@ def setup(self): # Add helper functions self.globals["FINAL_VAR"] = self._final_var + self.globals["SHOW_VARS"] = self._show_vars self.globals["llm_query"] = self._llm_query self.globals["llm_query_batched"] = self._llm_query_batched @@ -169,7 +170,27 @@ def _final_var(self, variable_name: str) -> str: variable_name = variable_name.strip().strip("\"'") if variable_name in self.locals: return str(self.locals[variable_name]) - return f"Error: Variable '{variable_name}' not found" + + # Provide helpful error message with available variables + available = [k for k in self.locals.keys() if not k.startswith("_")] + if available: + return ( + f"Error: Variable '{variable_name}' not found. " + f"Available variables: {available}. " + f"You must create and assign a variable BEFORE calling FINAL_VAR on it." + ) + return ( + f"Error: Variable '{variable_name}' not found. " + f"No variables have been created yet. " + f"You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it." + ) + + def _show_vars(self) -> str: + """Show all available variables in the REPL environment.""" + available = {k: type(v).__name__ for k, v in self.locals.items() if not k.startswith("_")} + if not available: + return "No variables created yet. Use ```repl``` blocks to create variables." + return f"Available variables: {available}" def _llm_query(self, prompt: str, model: str | None = None) -> str: """Query the LM via socket connection to the handler. diff --git a/rlm/environments/modal_repl.py b/rlm/environments/modal_repl.py index dc6801871..a00da7b2d 100644 --- a/rlm/environments/modal_repl.py +++ b/rlm/environments/modal_repl.py @@ -221,7 +221,16 @@ def FINAL_VAR(variable_name): variable_name = variable_name.strip().strip("\\"\\'") if variable_name in _locals: return str(_locals[variable_name]) - return f"Error: Variable '{{variable_name}}' not found" + available = [k for k in _locals.keys() if not k.startswith("_")] + if available: + return f"Error: Variable '{{variable_name}}' not found. Available variables: {{available}}. You must create and assign a variable BEFORE calling FINAL_VAR on it." + return f"Error: Variable '{{variable_name}}' not found. No variables have been created yet. You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it." + +def SHOW_VARS(): + available = {{k: type(v).__name__ for k, v in _locals.items() if not k.startswith("_")}} + if not available: + return "No variables created yet. Use ```repl``` blocks to create variables." + return f"Available variables: {{available}}" _globals = {{ "__builtins__": __builtins__, @@ -229,6 +238,7 @@ def FINAL_VAR(variable_name): "llm_query": llm_query, "llm_query_batched": llm_query_batched, "FINAL_VAR": FINAL_VAR, + "SHOW_VARS": SHOW_VARS, }} code = base64.b64decode("{code_b64}").decode() diff --git a/rlm/environments/prime_repl.py b/rlm/environments/prime_repl.py index aad4069a8..22aad1518 100644 --- a/rlm/environments/prime_repl.py +++ b/rlm/environments/prime_repl.py @@ -220,7 +220,16 @@ def FINAL_VAR(variable_name): variable_name = variable_name.strip().strip("\\"\\'") if variable_name in _locals: return str(_locals[variable_name]) - return f"Error: Variable '{{variable_name}}' not found" + available = [k for k in _locals.keys() if not k.startswith("_")] + if available: + return f"Error: Variable '{{variable_name}}' not found. Available variables: {{available}}. You must create and assign a variable BEFORE calling FINAL_VAR on it." + return f"Error: Variable '{{variable_name}}' not found. No variables have been created yet. You must create and assign a variable in a REPL block BEFORE calling FINAL_VAR on it." + +def SHOW_VARS(): + available = {{k: type(v).__name__ for k, v in _locals.items() if not k.startswith("_")}} + if not available: + return "No variables created yet. Use ```repl``` blocks to create variables." + return f"Available variables: {{available}}" _globals = {{ "__builtins__": __builtins__, @@ -228,6 +237,7 @@ def FINAL_VAR(variable_name): "llm_query": llm_query, "llm_query_batched": llm_query_batched, "FINAL_VAR": FINAL_VAR, + "SHOW_VARS": SHOW_VARS, }} code = base64.b64decode("{code_b64}").decode() diff --git a/rlm/utils/prompts.py b/rlm/utils/prompts.py index f69b22920..5e6bfdc93 100644 --- a/rlm/utils/prompts.py +++ b/rlm/utils/prompts.py @@ -10,7 +10,8 @@ 1. A `context` variable that contains extremely important information about your query. You should check the content of the `context` variable to understand what you are working with. Make sure you look through it sufficiently as you answer your query. 2. A `llm_query` function that allows you to query an LLM (that can handle around 500K chars) inside your REPL environment. 3. A `llm_query_batched` function that allows you to query multiple prompts concurrently: `llm_query_batched(prompts: List[str]) -> List[str]`. This is much faster than sequential `llm_query` calls when you have multiple independent queries. Results are returned in the same order as the input prompts. -4. The ability to use `print()` statements to view the output of your REPL code and continue your reasoning. +4. A `SHOW_VARS()` function that returns all variables you have created in the REPL. Use this to check what variables exist before using FINAL_VAR. +5. The ability to use `print()` statements to view the output of your REPL code and continue your reasoning. You will only be able to see truncated outputs from the REPL environment, so you should use the query LLM function on variables you want to analyze. You will find this function especially useful when you have to analyze the semantics of the context. Use these variables as buffers to build up your final answer. Make sure to explicitly look through the entire context in REPL before answering your query. An example strategy is to first look at the context and figure out a chunking strategy, then break up the context into smart chunks, and query an LLM per chunk with a particular question and save the answers to a buffer, then query an LLM with all the buffers to produce your final answer. @@ -76,6 +77,15 @@ 1. Use FINAL(your final answer here) to provide the answer directly 2. Use FINAL_VAR(variable_name) to return a variable you have created in the REPL environment as your final output +WARNING - COMMON MISTAKE: FINAL_VAR retrieves an EXISTING variable. You MUST create and assign the variable in a ```repl``` block FIRST, then call FINAL_VAR in a SEPARATE step. For example: +- WRONG: Calling FINAL_VAR(my_answer) without first creating `my_answer` in a repl block +- CORRECT: First run ```repl +my_answer = "the result" +print(my_answer) +``` then in the NEXT response call FINAL_VAR(my_answer) + +If you're unsure what variables exist, you can call SHOW_VARS() in a repl block to see all available variables. + Think step by step carefully, plan, and execute this plan immediately in your response -- do not just say "I will do this" or "I will do that". Output to the REPL environment and recursive LLMs as much as possible. Remember to explicitly answer the original query in your final answer. """ )