From f62bc6040ba9c32cea07eacf2bc7221c37c62726 Mon Sep 17 00:00:00 2001 From: TomuHirata Date: Thu, 15 May 2025 22:26:38 +0900 Subject: [PATCH 1/7] add CodeAct --- dspy/predict/__init__.py | 2 + dspy/predict/code_act.py | 117 +++++++++++++++++++++++++++++++++ tests/predict/test_code_act.py | 88 +++++++++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 dspy/predict/code_act.py create mode 100644 tests/predict/test_code_act.py diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py index 00fd80d760..f1f54c3375 100644 --- a/dspy/predict/__init__.py +++ b/dspy/predict/__init__.py @@ -8,6 +8,7 @@ from dspy.predict.predict import Predict from dspy.predict.program_of_thought import ProgramOfThought from dspy.predict.react import ReAct, Tool +from dspy.predict.code_act import CodeAct from dspy.predict.refine import Refine __all__ = [ @@ -15,6 +16,7 @@ "BestOfN", "ChainOfThought", "ChainOfThoughtWithHint", + "CodeAct", "KNN", "MultiChainComparison", "Predict", diff --git a/dspy/predict/code_act.py b/dspy/predict/code_act.py new file mode 100644 index 0000000000..9c1191b3ef --- /dev/null +++ b/dspy/predict/code_act.py @@ -0,0 +1,117 @@ + +from inspect import Signature +import logging +import inspect +from typing import Callable, Union + +from litellm import Type +import dspy +from dspy.primitives.python_interpreter import PythonInterpreter +from dspy.primitives.tool import Tool +from dspy.signatures.signature import ensure_signature +from dspy.predict.react import ReAct +from dspy.predict.program_of_thought import ProgramOfThought + +logger = logging.getLogger(__name__) + +class CodeAct(ReAct, ProgramOfThought): + """ + CodeAct is a module that utilizes the Code Interpreter and predefined tools to solve the problem. + """ + + def __init__(self, signature: Union[str, Type[Signature]], tools: list[Callable], max_iters=5): + """ + Initializes the CodeAct class with the specified model, temperature, and max tokens. + + Args: + signature (Union[str, Type[Signature]]): The signature of the module. + tools (list[Callable]): The tool callables to be used. CodeAct only accepts functions and not callable objects. + max_iters (int): The maximum number of iterations to generate the answer. + + Example: + + ```python + from dspy.predict import CodeAct + def factorial(n): + if n == 1: + return 1 + return n * factorial(n-1) + + act = CodeAct("n->factorial", tools=[factorial]) + act(n=5) # 120 + ``` + """ + self.signature = ensure_signature(signature) + self.max_iters = max_iters + + tools = [t if isinstance(t, Tool) else Tool(t) for t in tools] + if any( + not inspect.isfunction(tool.func) for tool in tools + ): + raise ValueError("CodeAct only accepts functions and not callable objects.") + tools = {tool.name: tool for tool in tools} + + instructions = self._build_instructions(self.signature, tools) + + codeact_signature = ( + dspy.Signature({**self.signature.input_fields}, "\n".join(instructions)) + .append("trajectory", dspy.InputField(), type_=str) + .append("generated_code", dspy.OutputField(desc="python code that answers the question"), type_=str) + .append("finished", dspy.OutputField(desc="a boolean flag to determine if the process is done"), type_=bool) + ) + + extract_signature = dspy.Signature( + {**self.signature.input_fields, **self.signature.output_fields}, + self.signature.instructions, + ).append("trajectory", dspy.InputField(), type_=str) + + self.tools: dict[str, Tool] = tools + self.codeact = dspy.Predict(codeact_signature) + self.extractor = dspy.ChainOfThought(extract_signature) + # It will raises exception when dspy cannot find available deno instance by now. + self.interpreter = PythonInterpreter() + + def _build_instructions(self, signature, tools): + instructions = [f"{signature.instructions}\n"] if signature.instructions else [] + inputs = ", ".join([f"`{k}`" for k in signature.input_fields.keys()]) + outputs = ", ".join([f"`{k}`" for k in signature.output_fields.keys()]) + + instructions.append( + f"You are an intelligent agent. For each episode, you will receive the fields {inputs} as input.\n" + f"Your goal is to generate executable Python code that collects any necessary information for producing {outputs}.\n" + f"For each iteration, you will generate a code snippet that either solves the task or progresses towards the solution.\n" + f"Ensure any output you wish to extract from the code is printed to the console. The code should be enclosed in a fenced code block.\n" + f"When all information for producing the outputs ({outputs}) are available to be extracted, mark `finished=True` besides the final Python code.\n" + f"You have access to the Python Standard Library and the following functions:" + ) + + for idx, tool in enumerate(tools.values()): + instructions.append(f"({idx + 1}) {tool}") + + return instructions + + def forward(self, **kwargs): + # Define the tool funcitons in the interpreter + for tool in self.tools.values(): + self.interpreter(inspect.getsource(tool.func)) + + trajectory = {} + max_iters = kwargs.pop("max_iters", self.max_iters) + for idx in range(max_iters): + code_data = self.codeact(trajectory=trajectory, **kwargs) + output = None + code, error = self._parse_code(code_data) + if not error: + output, error = self._execute_code(code) + + trajectory[f"generated_code_{idx}"] = code + trajectory[f"code_output_{idx}"] = output + else: + trajectory[f"observation_{idx}"] = f"Execution error in {error}" + + if code_data.finished: + break + + extract = self._call_with_potential_trajectory_truncation(self.extractor, trajectory, **kwargs) + self.interpreter.shutdown() + return dspy.Prediction(trajectory=trajectory, **extract) diff --git a/tests/predict/test_code_act.py b/tests/predict/test_code_act.py new file mode 100644 index 0000000000..7a6bc708cb --- /dev/null +++ b/tests/predict/test_code_act.py @@ -0,0 +1,88 @@ +import pytest +import shutil + +import dspy +from dspy import Signature +from dspy.predict import CodeAct +from dspy.utils import DummyLM + +# This test suite requires deno to be installed. Please install deno following https://docs.deno.com/runtime/getting_started/installation/ +is_deno_available = shutil.which("deno") is not None +skip_if_deno_not_available = pytest.mark.skipif( + not is_deno_available, reason="Deno is not installed or not in PATH" +) + + +class BasicQA(Signature): + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +def add(a: float, b: float) -> float: + "add two numbers" + return a + b + + +@skip_if_deno_not_available +def test_codeact_code_generation(): + lm = DummyLM( + [ + { + "reasoning": "Reason_A", + "generated_code": "```python\nresult = add(1,1)\nprint(result)\n```", + "finished": True, + }, + {"reasoning": "Reason_B", "answer": "2"}, + ] + ) + dspy.settings.configure(lm=lm) + program = CodeAct(BasicQA, tools=[add]) + res = program(question="What is 1+1?") + assert res.answer == "2" + assert res.trajectory == { + 'code_output_0': '"2\\n"', + 'generated_code_0': 'result = add(1,1)\nprint(result)', + } + assert program.interpreter.deno_process is None + + +class ExtremumFinder(Signature): + input_list = dspy.InputField() + maximum = dspy.OutputField(desc="The maximum of the given numbers") + minimum = dspy.OutputField(desc="The minimum of the given numbers") + +def extract_maximum_minimum(input_list: str) -> dict[str, float]: + numbers = list(map(float, input_list.split(","))) + return {"maximum": max(numbers), "minimum": min(numbers)} + +@skip_if_deno_not_available +def test_codeact_support_multiple_fields(): + lm = DummyLM( + [ + { + "reasoning": "Reason_A", + "generated_code": "```python\nresult = extract_maximum_minimum('2, 3, 5, 6')\nprint(result)\n```", + "finished": True, + }, + {"reasoning": "Reason_B", "maximum": "6", "minimum": "2"}, + ] + ) + dspy.settings.configure(lm=lm) + program = CodeAct(ExtremumFinder, tools=[extract_maximum_minimum]) + res = program(input_list="2, 3, 5, 6") + assert res.maximum == "6" + assert res.minimum == "2" + assert res.trajectory == { + 'code_output_0': '"{\'maximum\': 6.0, \'minimum\': 2.0}\\n"', + 'generated_code_0': "result = extract_maximum_minimum('2, 3, 5, 6')\nprint(result)", + } + assert program.interpreter.deno_process is None + +class CustomTool: + def __call__(self, a: float, b: float) -> float: + return a + b + +@skip_if_deno_not_available +def test_codeact_tool_validation(): + with pytest.raises(ValueError, match="CodeAct only accepts functions and not callable objects."): + CodeAct(BasicQA, tools=[CustomTool()]) From 5ab5d99536d0506f5b087183e0fb13805eba4414 Mon Sep 17 00:00:00 2001 From: TomuHirata Date: Thu, 15 May 2025 22:31:41 +0900 Subject: [PATCH 2/7] fix type --- dspy/predict/code_act.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dspy/predict/code_act.py b/dspy/predict/code_act.py index 9c1191b3ef..b643d71c3b 100644 --- a/dspy/predict/code_act.py +++ b/dspy/predict/code_act.py @@ -1,10 +1,9 @@ - -from inspect import Signature import logging import inspect -from typing import Callable, Union -from litellm import Type +from typing import Callable, Union, Type +from inspect import Signature + import dspy from dspy.primitives.python_interpreter import PythonInterpreter from dspy.primitives.tool import Tool @@ -29,7 +28,7 @@ def __init__(self, signature: Union[str, Type[Signature]], tools: list[Callable] max_iters (int): The maximum number of iterations to generate the answer. Example: - + ```python from dspy.predict import CodeAct def factorial(n): From 4b6ae687a2a4c1660eb1224040c5f56fe1326904 Mon Sep 17 00:00:00 2001 From: TomuHirata Date: Fri, 16 May 2025 13:08:34 +0900 Subject: [PATCH 3/7] change arg format --- dspy/predict/react.py | 1 + dspy/primitives/tool.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dspy/predict/react.py b/dspy/predict/react.py index f6098672a9..bad61c7d03 100644 --- a/dspy/predict/react.py +++ b/dspy/predict/react.py @@ -47,6 +47,7 @@ def __init__(self, signature, tools: list[Callable], max_iters=5): for idx, tool in enumerate(tools.values()): instr.append(f"({idx + 1}) {tool}") + instr.append("You should pass the tool argument in JSON format") react_signature = ( dspy.Signature({**signature.input_fields}, "\n".join(instr)) diff --git a/dspy/primitives/tool.py b/dspy/primitives/tool.py index 0bd19d9553..b8146b99d0 100644 --- a/dspy/primitives/tool.py +++ b/dspy/primitives/tool.py @@ -181,7 +181,7 @@ def __repr__(self): def __str__(self): desc = f", whose description is {self.desc}.".replace("\n", " ") if self.desc else "." - arg_desc = f"It takes arguments {self.args} in JSON format." + arg_desc = f"It takes arguments {self.args}." return f"{self.name}{desc} {arg_desc}" From be7b428f367cc735e7b0979da5a5244cd15ed55f Mon Sep 17 00:00:00 2001 From: TomuHirata Date: Fri, 16 May 2025 13:55:04 +0900 Subject: [PATCH 4/7] fix test --- tests/primitives/test_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/primitives/test_tool.py b/tests/primitives/test_tool.py index 1d1c203f7c..62d6086a98 100644 --- a/tests/primitives/test_tool.py +++ b/tests/primitives/test_tool.py @@ -253,7 +253,7 @@ def fn(x: int, **kwargs): def test_tool_str(): tool = Tool(dummy_function) - assert str(tool) == "dummy_function, whose description is A dummy function for testing. Args: x: An integer parameter y: A string parameter . It takes arguments {'x': {'type': 'integer'}, 'y': {'type': 'string', 'default': 'hello'}} in JSON format." + assert str(tool) == "dummy_function, whose description is A dummy function for testing. Args: x: An integer parameter y: A string parameter . It takes arguments {'x': {'type': 'integer'}, 'y': {'type': 'string', 'default': 'hello'}}." @pytest.mark.asyncio From 0aa73193e4fd58aa3d07aeeac40d34737e427b0a Mon Sep 17 00:00:00 2001 From: TomuHirata Date: Fri, 16 May 2025 18:50:42 +0900 Subject: [PATCH 5/7] remove unused f string --- dspy/predict/code_act.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dspy/predict/code_act.py b/dspy/predict/code_act.py index b643d71c3b..22010affe9 100644 --- a/dspy/predict/code_act.py +++ b/dspy/predict/code_act.py @@ -78,10 +78,10 @@ def _build_instructions(self, signature, tools): instructions.append( f"You are an intelligent agent. For each episode, you will receive the fields {inputs} as input.\n" f"Your goal is to generate executable Python code that collects any necessary information for producing {outputs}.\n" - f"For each iteration, you will generate a code snippet that either solves the task or progresses towards the solution.\n" - f"Ensure any output you wish to extract from the code is printed to the console. The code should be enclosed in a fenced code block.\n" + "For each iteration, you will generate a code snippet that either solves the task or progresses towards the solution.\n" + "Ensure any output you wish to extract from the code is printed to the console. The code should be enclosed in a fenced code block.\n" f"When all information for producing the outputs ({outputs}) are available to be extracted, mark `finished=True` besides the final Python code.\n" - f"You have access to the Python Standard Library and the following functions:" + "You have access to the Python Standard Library and the following functions:" ) for idx, tool in enumerate(tools.values()): From 7288cb4f3555847d0682e6c791b2954fa39937d1 Mon Sep 17 00:00:00 2001 From: TomuHirata Date: Mon, 19 May 2025 23:04:49 +0900 Subject: [PATCH 6/7] address comments --- dspy/predict/code_act.py | 13 ++++--- dspy/predict/react.py | 2 +- tests/predict/test_code_act.py | 63 ++++++++++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/dspy/predict/code_act.py b/dspy/predict/code_act.py index 22010affe9..109cd78714 100644 --- a/dspy/predict/code_act.py +++ b/dspy/predict/code_act.py @@ -100,13 +100,18 @@ def forward(self, **kwargs): code_data = self.codeact(trajectory=trajectory, **kwargs) output = None code, error = self._parse_code(code_data) - if not error: - output, error = self._execute_code(code) - trajectory[f"generated_code_{idx}"] = code + if error: + trajectory[f"observation_{idx}"] = f"Failed to parse the generated code: {error}" + continue + + trajectory[f"generated_code_{idx}"] = code + output, error = self._execute_code(code) + + if not error: trajectory[f"code_output_{idx}"] = output else: - trajectory[f"observation_{idx}"] = f"Execution error in {error}" + trajectory[f"observation_{idx}"] = f"Failed to execute the generated code: {error}" if code_data.finished: break diff --git a/dspy/predict/react.py b/dspy/predict/react.py index bad61c7d03..d0b7542fd5 100644 --- a/dspy/predict/react.py +++ b/dspy/predict/react.py @@ -47,7 +47,7 @@ def __init__(self, signature, tools: list[Callable], max_iters=5): for idx, tool in enumerate(tools.values()): instr.append(f"({idx + 1}) {tool}") - instr.append("You should pass the tool argument in JSON format") + instr.append("When providing `next_tool_args`, the value inside the field must be in JSON format") react_signature = ( dspy.Signature({**signature.input_fields}, "\n".join(instr)) diff --git a/tests/predict/test_code_act.py b/tests/predict/test_code_act.py index 7a6bc708cb..2565932648 100644 --- a/tests/predict/test_code_act.py +++ b/tests/predict/test_code_act.py @@ -17,12 +17,10 @@ class BasicQA(Signature): question = dspy.InputField() answer = dspy.OutputField(desc="often between 1 and 5 words") - def add(a: float, b: float) -> float: "add two numbers" return a + b - @skip_if_deno_not_available def test_codeact_code_generation(): lm = DummyLM( @@ -78,6 +76,67 @@ def test_codeact_support_multiple_fields(): } assert program.interpreter.deno_process is None + +@skip_if_deno_not_available +def test_codeact_code_parse_failure(): + lm = DummyLM( + [ + { + "reasoning": "Reason_A", + "generated_code": "```python\nparse(error\n```", + "finished": False, + }, + { + "reasoning": "Reason_A", + "generated_code": "```python\nresult = add(1,1)\nprint(result)\n```", + "finished": True, + }, + {"reasoning": "Reason_B", "answer": "2"}, + ] + ) + dspy.settings.configure(lm=lm) + program = CodeAct(BasicQA, tools=[add]) + res = program(question="What is 1+1?") + assert res.answer == "2" + assert res.trajectory == { + 'generated_code_0': 'parse(error', + 'observation_0': 'Failed to execute the generated code: Invalid Python syntax. message: ', + 'generated_code_1': 'result = add(1,1)\nprint(result)', + 'code_output_1': '"2\\n"', + } + assert program.interpreter.deno_process is None + + +@skip_if_deno_not_available +def test_codeact_code_execution_failure(): + lm = DummyLM( + [ + { + "reasoning": "Reason_A", + "generated_code": "```python\nunknown+1\n```", + "finished": False, + }, + { + "reasoning": "Reason_A", + "generated_code": "```python\nresult = add(1,1)\nprint(result)\n```", + "finished": True, + }, + {"reasoning": "Reason_B", "answer": "2"}, + ] + ) + dspy.settings.configure(lm=lm) + program = CodeAct(BasicQA, tools=[add]) + res = program(question="What is 1+1?") + assert res.answer == "2" + assert res.trajectory == { + 'generated_code_0': 'unknown+1', + 'observation_0': 'Failed to execute the generated code: NameError: ["name \'unknown\' is not defined"]', + 'generated_code_1': 'result = add(1,1)\nprint(result)', + 'code_output_1': '"2\\n"', + } + assert program.interpreter.deno_process is None + + class CustomTool: def __call__(self, a: float, b: float) -> float: return a + b From 192d0f2249f3f254c77dfd579f830fc96e46a77b Mon Sep 17 00:00:00 2001 From: TomuHirata Date: Tue, 20 May 2025 11:55:26 +0900 Subject: [PATCH 7/7] address comments --- dspy/predict/code_act.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspy/predict/code_act.py b/dspy/predict/code_act.py index 109cd78714..01b246916f 100644 --- a/dspy/predict/code_act.py +++ b/dspy/predict/code_act.py @@ -18,7 +18,7 @@ class CodeAct(ReAct, ProgramOfThought): CodeAct is a module that utilizes the Code Interpreter and predefined tools to solve the problem. """ - def __init__(self, signature: Union[str, Type[Signature]], tools: list[Callable], max_iters=5): + def __init__(self, signature: Union[str, Type[Signature]], tools: list[Callable], max_iters: int = 5): """ Initializes the CodeAct class with the specified model, temperature, and max tokens. @@ -55,7 +55,7 @@ def factorial(n): codeact_signature = ( dspy.Signature({**self.signature.input_fields}, "\n".join(instructions)) .append("trajectory", dspy.InputField(), type_=str) - .append("generated_code", dspy.OutputField(desc="python code that answers the question"), type_=str) + .append("generated_code", dspy.OutputField(desc="Python code that when executed, produces output relevant to answering the question"), type_=str) .append("finished", dspy.OutputField(desc="a boolean flag to determine if the process is done"), type_=bool) )