From 99427f8e2525f16168cfea02cb9938671bbcae9d Mon Sep 17 00:00:00 2001 From: eol Date: Sat, 16 May 2026 10:43:40 -0400 Subject: [PATCH 01/23] fix(avatar): remove deprecated prefix args from internal signatures (#9767) --- dspy/predict/avatar/signatures.py | 3 --- dspy/teleprompt/avatar_optimizer.py | 8 -------- 2 files changed, 11 deletions(-) diff --git a/dspy/predict/avatar/signatures.py b/dspy/predict/avatar/signatures.py index 65d1147521..13ac0cca9e 100644 --- a/dspy/predict/avatar/signatures.py +++ b/dspy/predict/avatar/signatures.py @@ -10,14 +10,11 @@ class Actor(dspy.Signature): Note: You can opt to use no tools and provide the final answer directly. You can also one tool multiple times with different input queries if applicable.""" goal: str = dspy.InputField( - prefix="Goal:", desc="Task to be accomplished.", ) tools: list[str] = dspy.InputField( - prefix="Tools:", desc="list of tools to use", ) action_1: Action = dspy.OutputField( - prefix="Action 1:", desc="1st action to take.", ) diff --git a/dspy/teleprompt/avatar_optimizer.py b/dspy/teleprompt/avatar_optimizer.py index ddba74e5f2..ea26270085 100644 --- a/dspy/teleprompt/avatar_optimizer.py +++ b/dspy/teleprompt/avatar_optimizer.py @@ -28,23 +28,18 @@ class Comparator(dspy.Signature): (3) Lastly, specify the modification in tools used that can lead to improved performance on the negative inputs.""" instruction: str = dspy.InputField( - prefix="Instruction: ", desc="Instruction for the actor to execute the task", ) actions: list[str] = dspy.InputField( - prefix="Actions: ", desc="Actions actor can take to complete the task", ) pos_input_with_metrics: list[EvalResult] = dspy.InputField( - prefix="Positive Inputs: ", desc="Positive inputs along with their score on a evaluation metric and actions taken", ) neg_input_with_metrics: list[EvalResult] = dspy.InputField( - prefix="Negative Inputs: ", desc="Negative inputs along with their score on a evaluation metric and actions taken", ) feedback: str = dspy.OutputField( - prefix="Feedback: ", desc="Feedback for the actor to improve the performance of negative inputs", ) @@ -59,15 +54,12 @@ class FeedbackBasedInstruction(dspy.Signature): Make sure that the new instruction talks about how to use the tools effectively and should be no more than 3 paragraphs long. The previous instruction contains general guidelines that you must retain in the new instruction.""" previous_instruction: str = dspy.InputField( - prefix="Previous Instruction: ", desc="Previous instruction for the actor to execute the task", ) feedback: str = dspy.InputField( - prefix="Feedback: ", desc="Feedback for the actor to improve the performance of negative inputs", ) new_instruction: str = dspy.OutputField( - prefix="New Instruction: ", desc="New instruction for the actor to execute the task", ) From 661a612cc64cac08af80ce58af52ec87d1a81f73 Mon Sep 17 00:00:00 2001 From: Abhishek Krishna Date: Sun, 17 May 2026 19:25:36 +0530 Subject: [PATCH 02/23] fix: correct mismatched closing code fence in README (#9771) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f36a4aa82..fe73d8cc89 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ To install the very latest from `main`: ```bash pip install git+https://github.com/stanfordnlp/dspy.git -```` +``` From 52efe77814e12b1d56e522bf5f8ac007271a874e Mon Sep 17 00:00:00 2001 From: Maxime Rivest Date: Thu, 21 May 2026 14:37:20 -0400 Subject: [PATCH 03/23] Add exact adapter format message tests (#9791) We did not have complete end-to-end coverage for the three adapters to ensure their rendering behavior stays stable over time. This PR adds exact-message regression tests for the chat, JSON, and XML adapters to help prevent adapter rendering drift. --- tests/adapters/test_chat_adapter.py | 98 ++++++++++++++++++ tests/adapters/test_json_adapter.py | 140 +++++++++++++++++++++++++ tests/adapters/test_xml_adapter.py | 152 ++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+) diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py index 86c662b3ba..cfa5466607 100644 --- a/tests/adapters/test_chat_adapter.py +++ b/tests/adapters/test_chat_adapter.py @@ -99,6 +99,104 @@ async def test_chat_adapter_async_call(): assert result == [{"answer": "Paris"}] +def test_chat_adapter_format_exact_messages_for_simple_signature(): + class QA(dspy.Signature): + """Answer the question.""" + + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages = dspy.ChatAdapter().format(QA, [], {"question": "What is the capital of France?"}) + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str): +All interactions will be structured in the following way, with the appropriate values filled in. + +[[ ## question ## ]] +{question} + +[[ ## answer ## ]] +{answer} + +[[ ## completed ## ]] +In adhering to this structure, your objective is:\x20 + Answer the question.""", + }, + { + "role": "user", + "content": """[[ ## question ## ]] +What is the capital of France? + +Respond with the corresponding output fields, starting with the field `[[ ## answer ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`.""", + }, + ] + + +def test_chat_adapter_format_exact_messages_with_demo_and_typed_outputs(): + class MultiAnswer(dspy.Signature): + """Answer the question with multiple answers and scores""" + + question: str = dspy.InputField() + answers: list[str] = dspy.OutputField() + scores: list[float] = dspy.OutputField() + + messages = dspy.ChatAdapter().format( + MultiAnswer, + demos=[{"question": "Q1", "answers": ["A1", "A2"], "scores": [0.1, 0.9]}], + inputs={"question": "Q2"}, + ) + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answers` (list[str]):\x20 +2. `scores` (list[float]): +All interactions will be structured in the following way, with the appropriate values filled in. + +[[ ## question ## ]] +{question} + +[[ ## answers ## ]] +{answers} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "string"}} + +[[ ## scores ## ]] +{scores} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "number"}} + +[[ ## completed ## ]] +In adhering to this structure, your objective is:\x20 + Answer the question with multiple answers and scores""", + }, + {"role": "user", "content": """[[ ## question ## ]] +Q1"""}, + { + "role": "assistant", + "content": """[[ ## answers ## ]] +["A1", "A2"] + +[[ ## scores ## ]] +[0.1, 0.9] + +[[ ## completed ## ]] +""", + }, + { + "role": "user", + "content": """[[ ## question ## ]] +Q2 + +Respond with the corresponding output fields, starting with the field `[[ ## answers ## ]]` (must be formatted as a valid Python list[str]), then `[[ ## scores ## ]]` (must be formatted as a valid Python list[float]), and then ending with the marker for `[[ ## completed ## ]]`.""", + }, + ] + + def test_chat_adapter_with_pydantic_models(): """ This test verifies that ChatAdapter can handle different input and output field types, both basic and nested. diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py index f376831d98..13ca0ea817 100644 --- a/tests/adapters/test_json_adapter.py +++ b/tests/adapters/test_json_adapter.py @@ -9,6 +9,146 @@ import dspy +def test_json_adapter_format_exact_messages_for_simple_signature(): + class StringSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages = dspy.JSONAdapter().format( + StringSignature, + demos=[], + inputs={"question": "What is the capital of France?"}, + ) + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str): +All interactions will be structured in the following way, with the appropriate values filled in. + +Inputs will have the following structure: + +[[ ## question ## ]] +{question} + +Outputs will be a JSON object with the following fields. + +{ + "answer": "{answer}" +} +In adhering to this structure, your objective is:\x20 + Given the fields `question`, produce the fields `answer`.""", + }, + { + "role": "user", + "content": """[[ ## question ## ]] +What is the capital of France? + +Respond with a JSON object in the following order of fields: `answer`.""", + }, + ] + + +def test_json_adapter_format_exact_messages_with_demo_and_typed_output(): + class MultiAnswer(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + confidence: float = dspy.OutputField() + + messages = dspy.JSONAdapter().format( + MultiAnswer, + demos=[{"question": "Q1", "answer": "A1", "confidence": 0.9}], + inputs={"question": "Q2"}, + ) + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str):\x20 +2. `confidence` (float): +All interactions will be structured in the following way, with the appropriate values filled in. + +Inputs will have the following structure: + +[[ ## question ## ]] +{question} + +Outputs will be a JSON object with the following fields. + +{ + "answer": "{answer}", + "confidence": "{confidence} # note: the value you produce must be a single float value" +} +In adhering to this structure, your objective is:\x20 + Given the fields `question`, produce the fields `answer`, `confidence`.""", + }, + {"role": "user", "content": """[[ ## question ## ]] +Q1"""}, + { + "role": "assistant", + "content": """{ + "answer": "A1", + "confidence": 0.9 +}""", + }, + { + "role": "user", + "content": """[[ ## question ## ]] +Q2 + +Respond with a JSON object in the following order of fields: `answer`, then `confidence` (must be formatted as a valid Python float).""", + }, + ] + + +def test_json_adapter_format_exact_messages_with_described_and_bool_outputs(): + class TestSignature(dspy.Signature): + input1: str = dspy.InputField() + output1: str = dspy.OutputField(desc="String output field") + output2: bool = dspy.OutputField() + + messages = dspy.JSONAdapter().format(TestSignature, [], {"input1": "Test input"}) + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `input1` (str): +Your output fields are: +1. `output1` (str): String output field +2. `output2` (bool): +All interactions will be structured in the following way, with the appropriate values filled in. + +Inputs will have the following structure: + +[[ ## input1 ## ]] +{input1} + +Outputs will be a JSON object with the following fields. + +{ + "output1": "{output1}", + "output2": "{output2} # note: the value you produce must be True or False" +} +In adhering to this structure, your objective is:\x20 + Given the fields `input1`, produce the fields `output1`, `output2`.""", + }, + { + "role": "user", + "content": """[[ ## input1 ## ]] +Test input + +Respond with a JSON object in the following order of fields: `output1`, then `output2` (must be formatted as a valid Python bool).""", + }, + ] + + def test_json_adapter_passes_structured_output_when_supported_by_model(): class OutputField3(pydantic.BaseModel): subfield1: int = pydantic.Field(description="Int subfield 1", ge=0, le=10) diff --git a/tests/adapters/test_xml_adapter.py b/tests/adapters/test_xml_adapter.py index ded864ad44..51857daff6 100644 --- a/tests/adapters/test_xml_adapter.py +++ b/tests/adapters/test_xml_adapter.py @@ -302,6 +302,158 @@ class QA(dspy.Signature): assert messages[1]["content"] == expected_user +def test_xml_adapter_format_exact_messages_for_simple_signature(): + class StringSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages = dspy.XMLAdapter().format( + StringSignature, + demos=[], + inputs={"question": "why did a chicken cross the kitchen?"}, + ) + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str): +All interactions will be structured in the following way, with the appropriate values filled in. + + +{question} + + + +{answer} + +In adhering to this structure, your objective is:\x20 + Given the fields `question`, produce the fields `answer`.""", + }, + { + "role": "user", + "content": """ +why did a chicken cross the kitchen? + + +Respond with the corresponding output fields wrapped in XML tags ``.""", + }, + ] + + +def test_xml_adapter_format_exact_messages_for_two_input_signature(): + class StringSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.InputField() + judgement: str = dspy.OutputField() + + messages = dspy.XMLAdapter().format( + StringSignature, + demos=[], + inputs={"question": "why did a chicken cross the kitchen?", "answer": "To get to the other side!"}, + ) + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str):\x20 +2. `answer` (str): +Your output fields are: +1. `judgement` (str): +All interactions will be structured in the following way, with the appropriate values filled in. + + +{question} + + + +{answer} + + + +{judgement} + +In adhering to this structure, your objective is:\x20 + Given the fields `question`, `answer`, produce the fields `judgement`.""", + }, + { + "role": "user", + "content": """ +why did a chicken cross the kitchen? + + + +To get to the other side! + + +Respond with the corresponding output fields wrapped in XML tags ``.""", + }, + ] + + +def test_xml_adapter_format_exact_messages_with_demo_and_typed_output(): + class MultiAnswer(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + score: float = dspy.OutputField() + + messages = dspy.XMLAdapter().format( + MultiAnswer, + demos=[{"question": "Q1", "answer": "A1", "score": 0.9}], + inputs={"question": "Q2"}, + ) + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str):\x20 +2. `score` (float): +All interactions will be structured in the following way, with the appropriate values filled in. + + +{question} + + + +{answer} + + + +{score} # note: the value you produce must be a single float value + +In adhering to this structure, your objective is:\x20 + Given the fields `question`, produce the fields `answer`, `score`.""", + }, + {"role": "user", "content": """ +Q1 +"""}, + { + "role": "assistant", + "content": """ +A1 + + + +0.9 +""", + }, + { + "role": "user", + "content": """ +Q2 + + +Respond with the corresponding output fields wrapped in XML tags ``, then ``.""", + }, + ] + + def test_format_system_message(): class MySignature(dspy.Signature): """Answer the question with multiple answers and scores""" From 6409a72302896c67d3e05c8c9268498b4f9a22c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 17:48:14 -0400 Subject: [PATCH 04/23] chore(deps): bump zizmorcore/zizmor-action from 0.5.3 to 0.5.5 (#9787) Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.5.3 to 0.5.5. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/b1d7e1fb5de872772f31590499237e7cce841e8e...a16621b09c6db4281f81a93cb393b05dcd7b7165) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.5.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index ba2e9d538c..c0add85326 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -21,7 +21,7 @@ jobs: with: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 + uses: zizmorcore/zizmor-action@a16621b09c6db4281f81a93cb393b05dcd7b7165 # v0.5.5 - name: Run actionlint uses: raven-actions/actionlint@205b530c5d9fa8f44ae9ed59f341a0db994aa6f8 # v2.1.2 From 6fb3f96cbbe28763054e6b15e6e509a344f386f8 Mon Sep 17 00:00:00 2001 From: Maxime Rivest Date: Thu, 21 May 2026 20:52:28 -0400 Subject: [PATCH 05/23] Expand exact adapter format message coverage (#9792) This PR expands the exact-format regression coverage for DSPy adapters. The goal is to lock current adapter rendering behavior before refactoring, including both the rendered messages and adapter-managed `lm_kwargs`. Coverage added: - More exact-message tests for `ChatAdapter`, `JSONAdapter`, and `XMLAdapter` - Exact-format tests for `BAMLAdapter` and `TwoStepAdapter` - Nested Pydantic input/output rendering - Complete and incomplete demos - Conversation history - Tools and native tool-calling `lm_kwargs` - Native reasoning `lm_kwargs` - Native citations behavior - Multimodal/custom adapter types: - `Image` - `Audio` - `File` - `Document` - `Citations` - `Code` - `Reasoning` - `Tool` - `ToolCalls` - custom `dspy.Type` - A kitchen-sink ChatAdapter regression test covering demos, history, custom types, nested Pydantic models, tools, literals, and scalar typed outputs. Testing: - Ran targeted adapter suite: `133 passed` --- tests/adapters/conftest.py | 55 + tests/adapters/test_baml_adapter.py | 79 ++ tests/adapters/test_chat_adapter.py | 1278 ++++++++++++++++++++++- tests/adapters/test_json_adapter.py | 537 +++++++++- tests/adapters/test_two_step_adapter.py | 52 + tests/adapters/test_xml_adapter.py | 314 +++++- 6 files changed, 2299 insertions(+), 16 deletions(-) create mode 100644 tests/adapters/conftest.py diff --git a/tests/adapters/conftest.py b/tests/adapters/conftest.py new file mode 100644 index 0000000000..e1d089dcee --- /dev/null +++ b/tests/adapters/conftest.py @@ -0,0 +1,55 @@ +import dspy + + +class StopAdapterCallCapture(BaseException): + """Stop adapter execution after capturing the LM call. + + The exact-format tests assert the adapter-to-LM boundary: the messages and + keyword arguments passed to the LM. Raising here avoids needing to craft a + parseable LM response for every signature under test. + """ + + +class CapturingLM(dspy.BaseLM): + def __init__(self, source_lm=None): + source_lm = source_lm or dspy.utils.DummyLM([{}]) + super().__init__( + model=source_lm.model, + model_type=source_lm.model_type, + cache=source_lm.cache, + **source_lm.kwargs, + ) + self.source_lm = source_lm + self.calls = [] + + @property + def supports_function_calling(self): + return self.source_lm.supports_function_calling + + @property + def supports_reasoning(self): + return self.source_lm.supports_reasoning + + @property + def supports_response_schema(self): + return self.source_lm.supports_response_schema + + @property + def supported_params(self): + return self.source_lm.supported_params + + def __call__(self, messages=None, **kwargs): + self.calls.append({"messages": messages, "kwargs": kwargs}) + raise StopAdapterCallCapture + + +def format_messages_and_lm_kwargs(adapter, signature, demos, inputs, lm_kwargs=None, lm=None): + capturing_lm = CapturingLM(lm) + try: + adapter(capturing_lm, dict(lm_kwargs or {}), signature, demos, inputs) + except StopAdapterCallCapture: + pass + + assert len(capturing_lm.calls) == 1 + call = capturing_lm.calls[0] + return call["messages"], call["kwargs"] diff --git a/tests/adapters/test_baml_adapter.py b/tests/adapters/test_baml_adapter.py index 5cc1481334..e8eaa9f6ee 100644 --- a/tests/adapters/test_baml_adapter.py +++ b/tests/adapters/test_baml_adapter.py @@ -8,6 +8,7 @@ import dspy from dspy.adapters.baml_adapter import COMMENT_SYMBOL, INDENTATION, BAMLAdapter +from tests.adapters.conftest import format_messages_and_lm_kwargs # Test fixtures - Pydantic models for testing @@ -51,6 +52,84 @@ class CircularModel(pydantic.BaseModel): field: "CircularModel" +def test_baml_adapter_format_exact_messages_for_simple_signature_with_demo(): + class QA(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(BAMLAdapter(), QA, [{"question": "Q1", "answer": "A1"}], {"question": "Q2"}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "Output field `answer` should be of type: string\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`."}, + {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, + {"role": "assistant", "content": '{\n "answer": "A1"\n}'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "Respond with a JSON object in the following order of fields: `answer`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_baml_adapter_format_exact_messages_with_nested_output(): + class BamlNested(pydantic.BaseModel): + value: int + tags: list[str] + + class TypedSignature(dspy.Signature): + question: str = dspy.InputField() + answer: BamlNested = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(BAMLAdapter(), TypedSignature, [], {"question": "Q"}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (BamlNested):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "Output field `answer` should be of type: {\n" + " value: int,\n" + " tags: string[],\n" + "}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q\n" + "\n" + "Respond with a JSON object in the following order of fields: `answer` (must be " + "formatted as a valid Python BamlNested)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + def test_baml_adapter_basic_schema_generation(): """Test that BAMLAdapter generates simplified schemas for Pydantic models.""" diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py index cfa5466607..17c1603766 100644 --- a/tests/adapters/test_chat_adapter.py +++ b/tests/adapters/test_chat_adapter.py @@ -1,3 +1,4 @@ +import re from typing import Literal from unittest import mock @@ -6,7 +7,8 @@ from litellm.utils import ChatCompletionMessageToolCall, Choices, Function, Message, ModelResponse import dspy -from dspy.experimental import Citations +from dspy.experimental import Citations, Document +from tests.adapters.conftest import format_messages_and_lm_kwargs @pytest.mark.parametrize( @@ -106,7 +108,10 @@ class QA(dspy.Signature): question: str = dspy.InputField() answer: str = dspy.OutputField() - messages = dspy.ChatAdapter().format(QA, [], {"question": "What is the capital of France?"}) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), QA, [], {"question": "What is the capital of France?"}) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs assert messages == [ { @@ -145,12 +150,15 @@ class MultiAnswer(dspy.Signature): answers: list[str] = dspy.OutputField() scores: list[float] = dspy.OutputField() - messages = dspy.ChatAdapter().format( + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), MultiAnswer, demos=[{"question": "Q1", "answers": ["A1", "A2"], "scores": [0.1, 0.9]}], inputs={"question": "Q2"}, ) + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + assert messages == [ { "role": "system", @@ -197,6 +205,1264 @@ class MultiAnswer(dspy.Signature): ] +def test_chat_adapter_format_exact_messages_with_nested_pydantic_models(): + class Address(pydantic.BaseModel): + city: str + country: str + + class Person(pydantic.BaseModel): + name: str + address: Address + tags: list[str] + + class Summary(pydantic.BaseModel): + headline: str + score: float + + class PydanticSignature(dspy.Signature): + person: Person = dspy.InputField() + summary: Summary = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + PydanticSignature, + [], + {"person": Person(name="Ada", address=Address(city="London", country="UK"), tags=["math", "code"])}, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `person` (Person):\n' + 'Your output fields are:\n' + '1. `summary` (Summary):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '[[ ## person ## ]]\n' + '{person}\n' + '\n' + '[[ ## summary ## ]]\n' + '{summary} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "properties": {"headline": {"type": "string", "title": ' + '"Headline"}, "score": {"type": "number", "title": "Score"}}, "required": ' + '["headline", "score"], "title": "Summary"}\n' + '\n' + '[[ ## completed ## ]]\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `person`, produce the fields `summary`.'}, + {"role": "user", + "content": '[[ ## person ## ]]\n' + '{"name": "Ada", "address": {"city": "London", "country": "UK"}, "tags": ["math", ' + '"code"]}\n' + '\n' + 'Respond with the corresponding output fields, starting with the field `[[ ## summary ' + '## ]]` (must be formatted as a valid Python Summary), and then ending with the ' + 'marker for `[[ ## completed ## ]]`.'}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_incomplete_demo(): + class IncompleteDemoSignature(dspy.Signature): + question: str = dspy.InputField() + context: str = dspy.InputField() + answer: str = dspy.OutputField() + confidence: float = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + IncompleteDemoSignature, + [{"question": "Q1", "answer": "A1"}], + {"question": "Q2", "context": "C2"}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str): \n" + "2. `context` (str):\n" + "Your output fields are:\n" + "1. `answer` (str): \n" + "2. `confidence` (float):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## context ## ]]\n" + "{context}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## confidence ## ]]\n" + "{confidence} # note: the value you produce must be a single float value\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `context`, produce the fields `answer`, " + "`confidence`."}, + {"role": "user", + "content": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## question ## ]]\n" + "Q1"}, + {"role": "assistant", + "content": "[[ ## answer ## ]]\n" + "A1\n" + "\n" + "[[ ## confidence ## ]]\n" + "Not supplied for this particular example.\n" + "\n" + "[[ ## completed ## ]]\n"}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "[[ ## context ## ]]\n" + "C2\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, then `[[ ## confidence ## ]]` (must be formatted as a valid Python float), " + "and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_history(): + class HistorySignature(dspy.Signature): + history: dspy.History = dspy.InputField() + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + history = dspy.History( + messages=[ + {"question": "What is 1+1?", "answer": "2"}, + {"question": "What is 2+2?", "answer": "4"}, + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + HistorySignature, + [], + {"history": history, "question": "What is 3+3?"}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `history` (History): \n" + "2. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## history ## ]]\n" + "{history}\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `history`, `question`, produce the fields `answer`."}, + {"role": "user", "content": "[[ ## question ## ]]\nWhat is 1+1?"}, + {"role": "assistant", "content": "[[ ## answer ## ]]\n2\n\n[[ ## completed ## ]]\n"}, + {"role": "user", "content": "[[ ## question ## ]]\nWhat is 2+2?"}, + {"role": "assistant", "content": "[[ ## answer ## ]]\n4\n\n[[ ## completed ## ]]\n"}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "What is 3+3?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_list_value_for_string_input(): + class ListAsStringSignature(dspy.Signature): + context: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), ListAsStringSignature, [], {"context": ["alpha", "beta"]}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `context` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## context ## ]]\n" + "{context}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `context`, produce the fields `answer`."}, + {"role": "user", + "content": "[[ ## context ## ]]\n" + "[1] «alpha»\n" + "[2] «beta»\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_literal_output(): + class LiteralSignature(dspy.Signature): + question: str = dspy.InputField() + verdict: Literal["yes", "no"] = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), LiteralSignature, [], {"question": "Is the sky blue?"}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `verdict` (Literal['yes', 'no']):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## verdict ## ]]\n" + "{verdict} # note: the value you produce must exactly match (no extra " + "characters) one of: yes; no\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `verdict`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Is the sky blue?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## verdict " + "## ]]` (must be formatted as a valid Python Literal['yes', 'no']), and then ending " + "with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_multimodal_custom_type_inputs(): + class CustomTypeSignature(dspy.Signature): + image: dspy.Image = dspy.InputField() + audio: dspy.Audio = dspy.InputField() + file: dspy.File = dspy.InputField() + document: Document = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + CustomTypeSignature, + [], + { + "image": dspy.Image("https://example.com/cat.png"), + "audio": dspy.Audio(data="QUJD", audio_format="wav"), + "file": dspy.File.from_file_id("file-123", filename="notes.txt"), + "document": Document(data="Alpha beta", title="Doc"), + }, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `image` (Image): \n" + "2. `audio` (Audio): \n" + "3. `file` (File): \n" + "4. `document` (Document): \n" + " Type description of Document: A document containing text content that can be " + "referenced and cited. Include the full text content and optionally a title for " + "proper referencing.\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## image ## ]]\n" + "{image}\n" + "\n" + "[[ ## audio ## ]]\n" + "{audio}\n" + "\n" + "[[ ## file ## ]]\n" + "{file}\n" + "\n" + "[[ ## document ## ]]\n" + "{document}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `image`, `audio`, `file`, `document`, produce the fields " + "`answer`."}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "text", "text": "\n\n[[ ## audio ## ]]\n"}, + {"type": "input_audio", "input_audio": {"data": "QUJD", "format": "wav"}}, + {"type": "text", "text": "\n\n[[ ## file ## ]]\n"}, + {"type": "file", "file": {"file_id": "file-123", "filename": "notes.txt"}}, + {"type": "text", "text": "\n\n[[ ## document ## ]]\n"}, + {"type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": "Alpha beta"}, + "citations": {"enabled": True}, + "title": "Doc"}, + {"type": "text", + "text": "\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ " + "## answer ## ]]`, and then ending with the marker for `[[ ## completed ## " + "]]`."}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_history_demo_pydantic_tools_and_image(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class Location(pydantic.BaseModel): + city: str + country: str + + class Profile(pydantic.BaseModel): + name: str + location: Location + interests: list[str] + + class AnswerCard(pydantic.BaseModel): + answer: str + sources: list[str] + + class RichRenderingSignature(dspy.Signature): + """Answer using all supplied context.""" + + history: dspy.History = dspy.InputField() + image: dspy.Image = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + profile: Profile = dspy.InputField() + question: str = dspy.InputField() + answer: AnswerCard = dspy.OutputField() + + tool = dspy.Tool(search) + demo_profile = Profile( + name="Ada", + location=Location(city="London", country="UK"), + interests=["math", "machines"], + ) + current_profile = Profile( + name="Grace", + location=Location(city="Arlington", country="USA"), + interests=["compilers", "navy"], + ) + history = dspy.History( + messages=[ + { + "profile": demo_profile, + "question": "Who is Ada?", + "answer": AnswerCard(answer="Ada is a mathematician.", sources=["memory"]), + } + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + RichRenderingSignature, + demos=[ + { + "image": dspy.Image("https://example.com/demo.png"), + "tools": [tool], + "profile": demo_profile, + "question": "What should we mention?", + "answer": AnswerCard(answer="Mention analytical engines.", sources=["demo"]), + } + ], + inputs={ + "history": history, + "image": dspy.Image("https://example.com/current.png"), + "tools": [tool], + "profile": current_profile, + "question": "What should the answer include?", + }, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `history` (History): \n' + '2. `image` (Image): \n' + '3. `tools` (list[Tool]): \n' + '4. `profile` (Profile): \n' + '5. `question` (str):\n' + 'Your output fields are:\n' + '1. `answer` (AnswerCard):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '[[ ## history ## ]]\n' + '{history}\n' + '\n' + '[[ ## image ## ]]\n' + '{image}\n' + '\n' + '[[ ## tools ## ]]\n' + '{tools}\n' + '\n' + '[[ ## profile ## ]]\n' + '{profile}\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + '[[ ## answer ## ]]\n' + '{answer} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "properties": {"answer": {"type": "string", "title": "Answer"}, ' + '"sources": {"type": "array", "items": {"type": "string"}, "title": "Sources"}}, ' + '"required": ["answer", "sources"], "title": "AnswerCard"}\n' + '\n' + '[[ ## completed ## ]]\n' + 'In adhering to this structure, your objective is: \n' + ' Answer using all supplied context.'}, + {"role": "user", + "content": [{"type": "text", + "text": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' + '"interests": ["math", "machines"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'What should we mention?'}]}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Mention analytical engines.", "sources": ["demo"]}\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, "interests": ' + '["math", "machines"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'Who is Ada?'}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Ada is a mathematician.", "sources": ["memory"]}\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' + '"interests": ["compilers", "navy"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'What should the answer include?\n' + '\n' + 'Respond with the corresponding output fields, starting with the field `[[ ' + '## answer ## ]]` (must be formatted as a valid Python AnswerCard), and ' + 'then ending with the marker for `[[ ## completed ## ]]`.'}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_base_custom_type_input(): + class Event(dspy.Type): + label: str + + def format(self): + return [{"type": "event", "event": {"label": self.label}}] + + @classmethod + def description(cls): + return "An event block." + + class EventSignature(dspy.Signature): + event: Event = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), EventSignature, [], {"event": Event(label="launch")}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `event` (Event): \n" + " Type description of Event: An event block.\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## event ## ]]\n" + "{event}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `event`, produce the fields `answer`."}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## event ## ]]\n"}, + {"type": "event", "event": {"label": "launch"}}, + {"type": "text", + "text": "\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ " + "## answer ## ]]`, and then ending with the marker for `[[ ## completed ## " + "]]`."}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_citations_output_demo(): + class CitationSignature(dspy.Signature): + question: str = dspy.InputField() + citations: Citations = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + CitationSignature, + [ + { + "question": "Q1", + "citations": Citations.from_dict_list( + [ + { + "cited_text": "alpha", + "document_index": 0, + "start_char_index": 0, + "end_char_index": 5, + } + ] + ), + } + ], + {"question": "Q2"}, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `citations` (Citations): \n' + ' Type description of Citations: Citations with quoted text and source references. ' + 'Include the exact text being cited and information about its source.\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + '[[ ## citations ## ]]\n' + '{citations} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "$defs": {"Citation": {"type": "object", "description": ' + '"Individual citation with character location information.", "properties": {"type": ' + '{"type": "string", "default": "char_location", "title": "Type"}, "cited_text": ' + '{"type": "string", "title": "Cited Text"}, "document_index": {"type": "integer", ' + '"title": "Document Index"}, "document_title": {"anyOf": [{"type": "string"}, ' + '{"type": "null"}], "default": null, "title": "Document Title"}, "end_char_index": ' + '{"type": "integer", "title": "End Char Index"}, "start_char_index": {"type": ' + '"integer", "title": "Start Char Index"}, "supported_text": {"anyOf": [{"type": ' + '"string"}, {"type": "null"}], "default": null, "title": "Supported Text"}}, ' + '"required": ["cited_text", "document_index", "start_char_index", "end_char_index"], ' + '"title": "Citation"}}, "description": "Experimental: This class may change or be ' + 'removed in a future release without warning (introduced in v3.0.4).\\n\\nCitations ' + 'extracted from an LM response with source references.\\n\\n This type represents ' + 'citations returned by language models that support\\n citation extraction, ' + "particularly Anthropic's Citations API through LiteLLM.\\n Citations include the " + 'quoted text and source information.\\n\\n Examples:\\n ```python\\n ' + 'import os\\n import dspy\\n from dspy.signatures import ' + 'Signature\\n from dspy.experimental import Citations, Document\\n ' + 'os.environ[\\"ANTHROPIC_API_KEY\\"] = \\"YOUR_ANTHROPIC_API_KEY\\"\\n\\n ' + "class AnswerWithSources(Signature):\\n '''Answer questions using provided " + "documents with citations.'''\\n documents: list[Document] = " + 'dspy.InputField()\\n question: str = dspy.InputField()\\n ' + 'answer: str = dspy.OutputField()\\n citations: Citations = ' + 'dspy.OutputField()\\n\\n # Create documents to provide as sources\\n ' + 'docs = [\\n Document(\\n data=\\"The Earth orbits the Sun ' + 'in an elliptical path.\\",\\n title=\\"Basic Astronomy ' + 'Facts\\"\\n ),\\n Document(\\n data=\\"Water ' + 'boils at 100°C at standard atmospheric pressure.\\",\\n ' + 'title=\\"Physics Fundamentals\\",\\n metadata={\\"author\\": \\"Dr. ' + 'Smith\\", \\"year\\": 2023}\\n )\\n ]\\n\\n # Use with a ' + 'model that supports citations like Claude\\n lm = ' + 'dspy.LM(\\"anthropic/claude-opus-4-1-20250805\\")\\n predictor = ' + 'dspy.Predict(AnswerWithSources)\\n result = predictor(documents=docs, ' + 'question=\\"What temperature does water boil?\\", lm=lm)\\n\\n for citation ' + 'in result.citations.citations:\\n print(citation.format())\\n ' + '```\\n ", "properties": {"citations": {"type": "array", "items": {"$ref": ' + '"#/$defs/Citation"}, "title": "Citations"}}, "required": ["citations"], "title": ' + '"Citations"}\n' + '\n' + '[[ ## completed ## ]]\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `citations`.'}, + {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, + {"role": "assistant", + "content": '[[ ## citations ## ]]\n' + '<>[{"type": "char_location", "cited_text": "alpha", ' + '"document_index": 0, "start_char_index": 0, "end_char_index": ' + '5}]<>\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## " + "citations ## ]]` (must be formatted as a valid Python Citations), and then ending " + "with the marker for `[[ ## completed ## ]]`."}] + def normalize_citations_schema_description(content): + return re.sub( + r'"description": ".*?", "properties":', + '"description": "", "properties":', + content, + ) + + messages[0]["content"] = normalize_citations_schema_description(messages[0]["content"]) + expected_messages[0]["content"] = normalize_citations_schema_description(expected_messages[0]["content"]) + + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_and_lm_kwargs_with_native_citations(): + class AnthropicLM(dspy.utils.DummyLM): + def __init__(self): + super().__init__([{}]) + self.model = "anthropic/claude-3-5-sonnet" + + class CitationSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + citations: Citations = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(), + CitationSignature, + [], + {"question": "Q?"}, + lm=AnthropicLM(), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`, `citations`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_preserves_passthrough_lm_kwargs(): + class PassthroughSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(), + PassthroughSignature, + [], + {"question": "Q?"}, + lm_kwargs={"temperature": 0.7, "max_tokens": 42, "stream": True, "cache": False}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {"temperature": 0.7, "max_tokens": 42, "stream": True, "cache": False} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_and_lm_kwargs_with_native_reasoning(): + class ReasoningLM(dspy.utils.DummyLM): + @property + def supports_reasoning(self): + return True + + class NativeReasoningSignature(dspy.Signature): + question: str = dspy.InputField() + reasoning: dspy.Reasoning = dspy.OutputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(), + NativeReasoningSignature, + [], + {"question": "Q?"}, + lm=ReasoningLM([{}]), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `reasoning`, `answer`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {"reasoning_effort": "low"} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_reasoning_and_code_outputs(): + python_code = dspy.Code["python"] + + class CodeSignature(dspy.Signature): + question: str = dspy.InputField() + reasoning: dspy.Reasoning = dspy.OutputField() + code: python_code = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + CodeSignature, + [{"question": "Q1", "reasoning": dspy.Reasoning(content="Think"), "code": python_code(code="print('hi')")}], + {"question": "Q2"}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `reasoning` (str): \n" + "2. `code` (Code_python): \n" + " Type description of Code_python: Code represented in a string, specified in the " + "`code` field. If this is an output field, the code field should follow the markdown " + "code block format, e.g. \n" + "```python\n" + "{code}\n" + "```\n" + "Programming language: python\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## reasoning ## ]]\n" + "{reasoning}\n" + "\n" + "[[ ## code ## ]]\n" + "{code}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `reasoning`, `code`."}, + {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, + {"role": "assistant", + "content": "[[ ## reasoning ## ]]\n" + "Think\n" + "\n" + "[[ ## code ## ]]\n" + "print('hi')\n" + "\n" + "[[ ## completed ## ]]\n"}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## " + "reasoning ## ]]` (must be formatted as a valid Python str), then `[[ ## code ## ]]` " + "(must be formatted as a valid Python Code_python), and then ending with the marker " + "for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_and_lm_kwargs_with_native_tool_calling(): + class FunctionCallingLM(dspy.utils.DummyLM): + @property + def supports_function_calling(self): + return True + + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class NativeToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(use_native_function_calling=True), + NativeToolSignature, + [], + {"question": "Q?", "tools": [dspy.Tool(search)]}, + lm=FunctionCallingLM([{}]), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?\n" + "\n" + "Respond with the corresponding output fields, starting with the field , and then " + "ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {"tools": [{"type": "function", + "function": {"name": "search", + "description": "Search for documents.", + "parameters": {"type": "object", + "properties": {"query": {"type": "string"}, + "k": {"type": "integer", "default": 3}}, + "required": ["query", "k"]}}}]} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_tool_input(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + ToolSignature, + [], + {"question": "Q?", "tools": [dspy.Tool(search)]}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str): \n" + "2. `tools` (list[Tool]):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## tools ## ]]\n" + "{tools}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `tools`, produce the fields `answer`."}, + {"role": "user", + "content": '[[ ## question ## ]]\n' + 'Q?\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It takes ' + "arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', 'default': " + '3}}."]\n' + '\n' + 'Respond with the corresponding output fields, starting with the field `[[ ## answer ' + '## ]]`, and then ending with the marker for `[[ ## completed ## ]]`.'}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_kitchen_sink(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class Event(dspy.Type): + label: str + + def format(self): + return [{"type": "event", "event": {"label": self.label}}] + + @classmethod + def description(cls): + return "An event block." + + class Location(pydantic.BaseModel): + city: str + country: str + + class Profile(pydantic.BaseModel): + name: str + location: Location + interests: list[str] + + class AnswerCard(pydantic.BaseModel): + answer: str + sources: list[str] + + class KitchenSinkSignature(dspy.Signature): + """Answer carefully using every available signal.""" + + history: dspy.History = dspy.InputField() + image: dspy.Image = dspy.InputField() + audio: dspy.Audio = dspy.InputField() + file: dspy.File = dspy.InputField() + document: Document = dspy.InputField() + event: Event = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + profile: Profile = dspy.InputField() + context: str = dspy.InputField() + question: str = dspy.InputField() + answer: AnswerCard = dspy.OutputField() + verdict: Literal["yes", "no"] = dspy.OutputField() + confidence: float = dspy.OutputField() + + tool = dspy.Tool(search) + demo_profile = Profile( + name="Ada", + location=Location(city="London", country="UK"), + interests=["math", "machines"], + ) + current_profile = Profile( + name="Grace", + location=Location(city="Arlington", country="USA"), + interests=["compilers", "navy"], + ) + history = dspy.History( + messages=[ + { + "profile": demo_profile, + "context": ["old note", "older note"], + "question": "Who is Ada?", + "answer": AnswerCard(answer="Ada is a mathematician.", sources=["memory"]), + "verdict": "yes", + "confidence": 0.8, + } + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + KitchenSinkSignature, + demos=[ + { + "image": dspy.Image("https://example.com/demo.png"), + "audio": dspy.Audio(data="REVNTw==", audio_format="wav"), + "file": dspy.File.from_file_id("file-demo", filename="demo.txt"), + "document": Document(data="Demo document", title="Demo Doc"), + "event": Event(label="demo-event"), + "tools": [tool], + "profile": demo_profile, + "context": ["demo context one", "demo context two"], + "question": "What should we mention?", + "answer": AnswerCard(answer="Mention analytical engines.", sources=["demo"]), + "verdict": "yes", + "confidence": 0.9, + }, + { + "question": "Incomplete example question", + "answer": AnswerCard(answer="Partial answer.", sources=["partial"]), + }, + ], + inputs={ + "history": history, + "image": dspy.Image("https://example.com/current.png"), + "audio": dspy.Audio(data="Q1VSUkVOVA==", audio_format="wav"), + "file": dspy.File.from_file_id("file-current", filename="current.txt"), + "document": Document(data="Current document", title="Current Doc"), + "event": Event(label="current-event"), + "tools": [tool], + "profile": current_profile, + "context": ["current context one", "current context two"], + "question": "What should the answer include?", + }, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `history` (History): \n' + '2. `image` (Image): \n' + '3. `audio` (Audio): \n' + '4. `file` (File): \n' + '5. `document` (Document): \n' + ' Type description of Document: A document containing text content that can be ' + 'referenced and cited. Include the full text content and optionally a title for ' + 'proper referencing.\n' + '6. `event` (Event): \n' + ' Type description of Event: An event block.\n' + '7. `tools` (list[Tool]): \n' + '8. `profile` (Profile): \n' + '9. `context` (str): \n' + '10. `question` (str):\n' + 'Your output fields are:\n' + '1. `answer` (AnswerCard): \n' + "2. `verdict` (Literal['yes', 'no']): \n" + '3. `confidence` (float):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '[[ ## history ## ]]\n' + '{history}\n' + '\n' + '[[ ## image ## ]]\n' + '{image}\n' + '\n' + '[[ ## audio ## ]]\n' + '{audio}\n' + '\n' + '[[ ## file ## ]]\n' + '{file}\n' + '\n' + '[[ ## document ## ]]\n' + '{document}\n' + '\n' + '[[ ## event ## ]]\n' + '{event}\n' + '\n' + '[[ ## tools ## ]]\n' + '{tools}\n' + '\n' + '[[ ## profile ## ]]\n' + '{profile}\n' + '\n' + '[[ ## context ## ]]\n' + '{context}\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + '[[ ## answer ## ]]\n' + '{answer} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "properties": {"answer": {"type": "string", "title": "Answer"}, ' + '"sources": {"type": "array", "items": {"type": "string"}, "title": "Sources"}}, ' + '"required": ["answer", "sources"], "title": "AnswerCard"}\n' + '\n' + '[[ ## verdict ## ]]\n' + '{verdict} # note: the value you produce must exactly match (no extra ' + 'characters) one of: yes; no\n' + '\n' + '[[ ## confidence ## ]]\n' + '{confidence} # note: the value you produce must be a single float value\n' + '\n' + '[[ ## completed ## ]]\n' + 'In adhering to this structure, your objective is: \n' + ' Answer carefully using every available signal.'}, + {"role": "user", + "content": [{"type": "text", + "text": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, + {"type": "text", "text": "\n\n[[ ## audio ## ]]\n"}, + {"type": "input_audio", "input_audio": {"data": "REVNTw==", "format": "wav"}}, + {"type": "text", "text": "\n\n[[ ## file ## ]]\n"}, + {"type": "file", "file": {"file_id": "file-demo", "filename": "demo.txt"}}, + {"type": "text", "text": "\n\n[[ ## document ## ]]\n"}, + {"type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": "Demo document"}, + "citations": {"enabled": True}, + "title": "Demo Doc"}, + {"type": "text", "text": "\n\n[[ ## event ## ]]\n"}, + {"type": "event", "event": {"label": "demo-event"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' + '"interests": ["math", "machines"]}\n' + '\n' + '[[ ## context ## ]]\n' + '[1] «demo context one»\n' + '[2] «demo context two»\n' + '\n' + '[[ ## question ## ]]\n' + 'What should we mention?'}]}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Mention analytical engines.", "sources": ["demo"]}\n' + '\n' + '[[ ## verdict ## ]]\n' + 'yes\n' + '\n' + '[[ ## confidence ## ]]\n' + '0.9\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## question ## ]]\n" + "Incomplete example question"}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Partial answer.", "sources": ["partial"]}\n' + '\n' + '[[ ## verdict ## ]]\n' + 'Not supplied for this particular example. \n' + '\n' + '[[ ## confidence ## ]]\n' + 'Not supplied for this particular example.\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, "interests": ' + '["math", "machines"]}\n' + '\n' + '[[ ## context ## ]]\n' + '[1] «old note»\n' + '[2] «older note»\n' + '\n' + '[[ ## question ## ]]\n' + 'Who is Ada?'}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Ada is a mathematician.", "sources": ["memory"]}\n' + '\n' + '[[ ## verdict ## ]]\n' + 'yes\n' + '\n' + '[[ ## confidence ## ]]\n' + '0.8\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, + {"type": "text", "text": "\n\n[[ ## audio ## ]]\n"}, + {"type": "input_audio", "input_audio": {"data": "Q1VSUkVOVA==", "format": "wav"}}, + {"type": "text", "text": "\n\n[[ ## file ## ]]\n"}, + {"type": "file", "file": {"file_id": "file-current", "filename": "current.txt"}}, + {"type": "text", "text": "\n\n[[ ## document ## ]]\n"}, + {"type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": "Current document"}, + "citations": {"enabled": True}, + "title": "Current Doc"}, + {"type": "text", "text": "\n\n[[ ## event ## ]]\n"}, + {"type": "event", "event": {"label": "current-event"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' + '"interests": ["compilers", "navy"]}\n' + '\n' + '[[ ## context ## ]]\n' + '[1] «current context one»\n' + '[2] «current context two»\n' + '\n' + '[[ ## question ## ]]\n' + 'What should the answer include?\n' + '\n' + 'Respond with the corresponding output fields, starting with the field `[[ ' + '## answer ## ]]` (must be formatted as a valid Python AnswerCard), then ' + "`[[ ## verdict ## ]]` (must be formatted as a valid Python Literal['yes', " + "'no']), then `[[ ## confidence ## ]]` (must be formatted as a valid Python " + 'float), and then ending with the marker for `[[ ## completed ## ]]`.'}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + def test_chat_adapter_with_pydantic_models(): """ This test verifies that ChatAdapter can handle different input and output field types, both basic and nested. @@ -293,7 +1559,7 @@ def test_chat_adapter_exception_raised_on_failure(): signature = dspy.make_signature("question->answer") adapter = dspy.ChatAdapter() invalid_completion = "{'output':'mismatched value'}" - with pytest.raises(dspy.utils.exceptions.AdapterParseError, match="Adapter ChatAdapter failed to parse*"): + with pytest.raises(dspy.utils.exceptions.AdapterParseError, match=r"Adapter ChatAdapter failed to parse.*"): adapter.parse(signature, invalid_completion) @@ -824,7 +2090,7 @@ class MySignature(dspy.Signature): expected_system_message = """Your input fields are: 1. `question` (str): Your output fields are: -1. `answers` (list[str]): +1. `answers` (list[str]):\x20 2. `scores` (list[float]): All interactions will be structured in the following way, with the appropriate values filled in. @@ -838,7 +2104,7 @@ class MySignature(dspy.Signature): {scores} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "number"}} [[ ## completed ## ]] -In adhering to this structure, your objective is: +In adhering to this structure, your objective is:\x20 Answer the question with multiple answers and scores""" assert system_message == expected_system_message diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py index 13ca0ea817..c6dd1f679d 100644 --- a/tests/adapters/test_json_adapter.py +++ b/tests/adapters/test_json_adapter.py @@ -1,3 +1,5 @@ +import enum +from typing import Literal from unittest import mock import pydantic @@ -7,6 +9,7 @@ from openai.types.responses import ResponseOutputMessage import dspy +from tests.adapters.conftest import format_messages_and_lm_kwargs def test_json_adapter_format_exact_messages_for_simple_signature(): @@ -14,12 +17,15 @@ class StringSignature(dspy.Signature): question: str = dspy.InputField() answer: str = dspy.OutputField() - messages = dspy.JSONAdapter().format( + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), StringSignature, demos=[], inputs={"question": "What is the capital of France?"}, ) + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + assert messages == [ { "role": "system", @@ -58,12 +64,15 @@ class MultiAnswer(dspy.Signature): answer: str = dspy.OutputField() confidence: float = dspy.OutputField() - messages = dspy.JSONAdapter().format( + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), MultiAnswer, demos=[{"question": "Q1", "answer": "A1", "confidence": 0.9}], inputs={"question": "Q2"}, ) + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + assert messages == [ { "role": "system", @@ -113,7 +122,10 @@ class TestSignature(dspy.Signature): output1: str = dspy.OutputField(desc="String output field") output2: bool = dspy.OutputField() - messages = dspy.JSONAdapter().format(TestSignature, [], {"input1": "Test input"}) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), TestSignature, [], {"input1": "Test input"}) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs assert messages == [ { @@ -149,6 +161,521 @@ class TestSignature(dspy.Signature): ] +def test_json_adapter_format_exact_messages_with_history_demo_pydantic_tools_and_image(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class Location(pydantic.BaseModel): + city: str + country: str + + class Profile(pydantic.BaseModel): + name: str + location: Location + interests: list[str] + + class AnswerCard(pydantic.BaseModel): + answer: str + sources: list[str] + + class RichRenderingSignature(dspy.Signature): + """Answer using all supplied context.""" + + history: dspy.History = dspy.InputField() + image: dspy.Image = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + profile: Profile = dspy.InputField() + question: str = dspy.InputField() + answer: AnswerCard = dspy.OutputField() + + tool = dspy.Tool(search) + demo_profile = Profile( + name="Ada", + location=Location(city="London", country="UK"), + interests=["math", "machines"], + ) + current_profile = Profile( + name="Grace", + location=Location(city="Arlington", country="USA"), + interests=["compilers", "navy"], + ) + history = dspy.History( + messages=[ + { + "profile": demo_profile, + "question": "Who is Ada?", + "answer": AnswerCard(answer="Ada is a mathematician.", sources=["memory"]), + } + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), + RichRenderingSignature, + demos=[ + { + "image": dspy.Image("https://example.com/demo.png"), + "tools": [tool], + "profile": demo_profile, + "question": "What should we mention?", + "answer": AnswerCard(answer="Mention analytical engines.", sources=["demo"]), + } + ], + inputs={ + "history": history, + "image": dspy.Image("https://example.com/current.png"), + "tools": [tool], + "profile": current_profile, + "question": "What should the answer include?", + }, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `history` (History): \n' + '2. `image` (Image): \n' + '3. `tools` (list[Tool]): \n' + '4. `profile` (Profile): \n' + '5. `question` (str):\n' + 'Your output fields are:\n' + '1. `answer` (AnswerCard):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## history ## ]]\n' + '{history}\n' + '\n' + '[[ ## image ## ]]\n' + '{image}\n' + '\n' + '[[ ## tools ## ]]\n' + '{tools}\n' + '\n' + '[[ ## profile ## ]]\n' + '{profile}\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "answer": "{answer} # note: the value you produce must adhere to the JSON ' + 'schema: {\\"type\\": \\"object\\", \\"properties\\": {\\"answer\\": {\\"type\\": ' + '\\"string\\", \\"title\\": \\"Answer\\"}, \\"sources\\": {\\"type\\": \\"array\\", ' + '\\"items\\": {\\"type\\": \\"string\\"}, \\"title\\": \\"Sources\\"}}, ' + '\\"required\\": [\\"answer\\", \\"sources\\"], \\"title\\": \\"AnswerCard\\"}"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Answer using all supplied context.'}, + {"role": "user", + "content": [{"type": "text", + "text": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' + '"interests": ["math", "machines"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'What should we mention?'}]}, + {"role": "assistant", + "content": '{\n' + ' "answer": {\n' + ' "answer": "Mention analytical engines.",\n' + ' "sources": [\n' + ' "demo"\n' + ' ]\n' + ' }\n' + '}'}, + {"role": "user", + "content": '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, "interests": ' + '["math", "machines"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'Who is Ada?'}, + {"role": "assistant", + "content": '{\n' + ' "answer": {\n' + ' "answer": "Ada is a mathematician.",\n' + ' "sources": [\n' + ' "memory"\n' + ' ]\n' + ' }\n' + '}'}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' + '"interests": ["compilers", "navy"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'What should the answer include?\n' + '\n' + 'Respond with a JSON object in the following order of fields: `answer` ' + '(must be formatted as a valid Python AnswerCard).'}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_json_adapter_format_exact_messages_with_int_and_mapping_outputs(): + class IntDictSignature(dspy.Signature): + question: str = dspy.InputField() + count: int = dspy.OutputField() + metadata: dict[str, int] = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), IntDictSignature, [], {"question": "Count things"}) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `count` (int): \n' + '2. `metadata` (dict[str, int]):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "count": "{count} # note: the value you produce must be a single int ' + 'value",\n' + ' "metadata": "{metadata} # note: the value you produce must adhere to the ' + 'JSON schema: {\\"type\\": \\"object\\", \\"additionalProperties\\": {\\"type\\": ' + '\\"integer\\"}}"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `count`, `metadata`.'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Count things\n" + "\n" + "Respond with a JSON object in the following order of fields: `count` (must be " + "formatted as a valid Python int), then `metadata` (must be formatted as a valid " + "Python dict[str, int])."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_json_adapter_format_exact_messages_with_literal_and_enum_outputs(): + class Label(enum.Enum): + POSITIVE = "positive" + NEGATIVE = "negative" + + class LiteralEnumSignature(dspy.Signature): + text: str = dspy.InputField() + decision: Literal["accept", "reject"] = dspy.OutputField() + label: Label = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), LiteralEnumSignature, [], {"text": "Looks good"}) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `text` (str):\n' + 'Your output fields are:\n' + "1. `decision` (Literal['accept', 'reject']): \n" + '2. `label` (Label):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## text ## ]]\n' + '{text}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "decision": "{decision} # note: the value you produce must exactly match ' + '(no extra characters) one of: accept; reject",\n' + ' "label": "{label} # note: the value you produce must be one of: positive; ' + 'negative"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `text`, produce the fields `decision`, `label`.'}, + {"role": "user", + "content": "[[ ## text ## ]]\n" + "Looks good\n" + "\n" + "Respond with a JSON object in the following order of fields: `decision` (must be " + "formatted as a valid Python Literal['accept', 'reject']), then `label` (must be " + "formatted as a valid Python Label)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_json_adapter_format_exact_messages_with_nested_pydantic_output(): + class JsonNestedAddress(pydantic.BaseModel): + city: str + country: str + + class JsonNestedSummary(pydantic.BaseModel): + title: str + address: JsonNestedAddress + scores: list[float] + + class PydanticSignature(dspy.Signature): + question: str = dspy.InputField() + summary: JsonNestedSummary = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), PydanticSignature, [], {"question": "Summarize"}) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `summary` (JsonNestedSummary):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "summary": "{summary} # note: the value you produce must adhere to the JSON ' + 'schema: {\\"type\\": \\"object\\", \\"$defs\\": {\\"JsonNestedAddress\\": ' + '{\\"type\\": \\"object\\", \\"properties\\": {\\"city\\": {\\"type\\": \\"string\\", ' + '\\"title\\": \\"City\\"}, \\"country\\": {\\"type\\": \\"string\\", \\"title\\": ' + '\\"Country\\"}}, \\"required\\": [\\"city\\", \\"country\\"], \\"title\\": ' + '\\"JsonNestedAddress\\"}}, \\"properties\\": {\\"address\\": {\\"$ref\\": ' + '\\"#/$defs/JsonNestedAddress\\"}, \\"scores\\": {\\"type\\": \\"array\\", ' + '\\"items\\": {\\"type\\": \\"number\\"}, \\"title\\": \\"Scores\\"}, \\"title\\": ' + '{\\"type\\": \\"string\\", \\"title\\": \\"Title\\"}}, \\"required\\": [\\"title\\", ' + '\\"address\\", \\"scores\\"], \\"title\\": \\"JsonNestedSummary\\"}"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `summary`.'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Summarize\n" + "\n" + "Respond with a JSON object in the following order of fields: `summary` (must be " + "formatted as a valid Python JsonNestedSummary)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_json_adapter_format_exact_messages_with_incomplete_demo(): + class IncompleteDemoSignature(dspy.Signature): + question: str = dspy.InputField() + context: str = dspy.InputField() + answer: str = dspy.OutputField() + score: float = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), + IncompleteDemoSignature, + [{"question": "Q1", "answer": "A1"}], + {"question": "Q2", "context": "C2"}, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str): \n' + '2. `context` (str):\n' + 'Your output fields are:\n' + '1. `answer` (str): \n' + '2. `score` (float):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + '[[ ## context ## ]]\n' + '{context}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "answer": "{answer}",\n' + ' "score": "{score} # note: the value you produce must be a single float ' + 'value"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, `context`, produce the fields `answer`, ' + '`score`.'}, + {"role": "user", + "content": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## question ## ]]\n" + "Q1"}, + {"role": "assistant", + "content": '{\n "answer": "A1",\n "score": "Not supplied for this particular example. "\n}'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "[[ ## context ## ]]\n" + "C2\n" + "\n" + "Respond with a JSON object in the following order of fields: `answer`, then `score` " + "(must be formatted as a valid Python float)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_json_adapter_format_exact_messages_and_lm_kwargs_with_native_tool_calling(): + class FunctionCallingLM(dspy.utils.DummyLM): + @property + def supports_function_calling(self): + return True + + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class NativeToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.JSONAdapter(use_native_function_calling=True), + NativeToolSignature, + [], + {"question": "Q?", "tools": [dspy.Tool(search)]}, + lm=FunctionCallingLM([{}]), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "Inputs will have the following structure:\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "Outputs will be a JSON object with the following fields.\n" + "\n" + "{}\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?\n" + "\n" + "Respond with a JSON object in the following order of fields: ."}] + assert messages == expected_messages + expected_lm_kwargs = {"tools": [{"type": "function", + "function": {"name": "search", + "description": "Search for documents.", + "parameters": {"type": "object", + "properties": {"query": {"type": "string"}, + "k": {"type": "integer", "default": 3}}, + "required": ["query", "k"]}}}]} + assert lm_kwargs == expected_lm_kwargs + +def test_json_adapter_format_exact_messages_with_tool_calls_output_demo(): + class ToolCallsSignature(dspy.Signature): + question: str = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(use_native_function_calling=False), + ToolCallsSignature, + [{"question": "Q1", "tool_calls": dspy.ToolCalls.from_dict_list([{"name": "search", "args": {"query": "cats"}}])}], + {"question": "Q2"}, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `tool_calls` (ToolCalls): \n' + ' Type description of ToolCalls: Tool calls information, including the name of the ' + 'tools and the arguments to be passed to it. Arguments must be provided in JSON ' + 'format.\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "tool_calls": "{tool_calls} # note: the value you produce must adhere to ' + 'the JSON schema: {\\"type\\": \\"object\\", \\"$defs\\": {\\"ToolCall\\": ' + '{\\"type\\": \\"object\\", \\"properties\\": {\\"args\\": {\\"type\\": \\"object\\", ' + '\\"additionalProperties\\": true, \\"title\\": \\"Args\\"}, \\"name\\": {\\"type\\": ' + '\\"string\\", \\"title\\": \\"Name\\"}}, \\"required\\": [\\"name\\", \\"args\\"], ' + '\\"title\\": \\"ToolCall\\"}}, \\"properties\\": {\\"tool_calls\\": {\\"type\\": ' + '\\"array\\", \\"items\\": {\\"$ref\\": \\"#/$defs/ToolCall\\"}, \\"title\\": \\"Tool ' + 'Calls\\"}}, \\"required\\": [\\"tool_calls\\"], \\"title\\": \\"ToolCalls\\"}"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `tool_calls`.'}, + {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, + {"role": "assistant", + "content": '{\n' + ' "tool_calls": {\n' + ' "tool_calls": [\n' + ' {\n' + ' "type": "function",\n' + ' "function": {\n' + ' "name": "search",\n' + ' "arguments": {\n' + ' "query": "cats"\n' + ' }\n' + ' }\n' + ' }\n' + ' ]\n' + ' }\n' + '}'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "Respond with a JSON object in the following order of fields: `tool_calls` (must be " + "formatted as a valid Python ToolCalls)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + def test_json_adapter_passes_structured_output_when_supported_by_model(): class OutputField3(pydantic.BaseModel): subfield1: int = pydantic.Field(description="Int subfield 1", ge=0, le=10) @@ -1138,7 +1665,7 @@ class MySignature(dspy.Signature): expected_system_message = """Your input fields are: 1. `question` (str): Your output fields are: -1. `answers` (list[str]): +1. `answers` (list[str]):\x20 2. `scores` (list[float]): All interactions will be structured in the following way, with the appropriate values filled in. @@ -1153,6 +1680,6 @@ class MySignature(dspy.Signature): "answers": "{answers} # note: the value you produce must adhere to the JSON schema: {\\"type\\": \\"array\\", \\"items\\": {\\"type\\": \\"string\\"}}", "scores": "{scores} # note: the value you produce must adhere to the JSON schema: {\\"type\\": \\"array\\", \\"items\\": {\\"type\\": \\"number\\"}}" } -In adhering to this structure, your objective is: +In adhering to this structure, your objective is:\x20 Answer the question with multiple answers and scores""" assert system_message == expected_system_message diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.py index f045188fbc..15a35fd30a 100644 --- a/tests/adapters/test_two_step_adapter.py +++ b/tests/adapters/test_two_step_adapter.py @@ -3,6 +3,58 @@ import pytest import dspy +from tests.adapters.conftest import format_messages_and_lm_kwargs + + +def test_two_step_adapter_format_exact_messages_for_simple_signature_with_demo(): + class QA(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.TwoStepAdapter(dspy.utils.DummyLM([{"answer": "x"}])) + messages, lm_kwargs = format_messages_and_lm_kwargs(adapter, QA, [{"question": "Q1", "answer": "A1"}], {"question": "Q2"}) + + expected_messages = [{"role": "system", + "content": "You are a helpful assistant that can solve tasks based on user input.\n" + "As input, you will be provided with:\n" + "1. `question` (str):\n" + "Your outputs must contain:\n" + "1. `answer` (str):\n" + "You should lay out your outputs in detail so that your answer can be understood by " + "another agent\n" + "Specific instructions: Given the fields `question`, produce the fields `answer`."}, + {"role": "user", "content": "question: Q1"}, + {"role": "assistant", "content": "answer: A1"}, + {"role": "user", "content": "question: Q2"}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_two_step_adapter_format_exact_messages_with_typed_outputs(): + class TypedSignature(dspy.Signature): + question: str = dspy.InputField() + count: int = dspy.OutputField() + answer: str = dspy.OutputField() + + adapter = dspy.TwoStepAdapter(dspy.utils.DummyLM([{"count": 1, "answer": "x"}])) + messages, lm_kwargs = format_messages_and_lm_kwargs(adapter, TypedSignature, [], {"question": "Q"}) + + expected_messages = [{"role": "system", + "content": "You are a helpful assistant that can solve tasks based on user input.\n" + "As input, you will be provided with:\n" + "1. `question` (str):\n" + "Your outputs must contain:\n" + "1. `count` (int): \n" + "2. `answer` (str):\n" + "You should lay out your outputs in detail so that your answer can be understood by " + "another agent\n" + "Specific instructions: Given the fields `question`, produce the fields `count`, " + "`answer`."}, + {"role": "user", "content": "question: Q"}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs def test_two_step_adapter_call(): diff --git a/tests/adapters/test_xml_adapter.py b/tests/adapters/test_xml_adapter.py index 51857daff6..ee300ba3cd 100644 --- a/tests/adapters/test_xml_adapter.py +++ b/tests/adapters/test_xml_adapter.py @@ -8,6 +8,7 @@ import dspy from dspy.adapters.chat_adapter import FieldInfoWithName from dspy.adapters.xml_adapter import XMLAdapter +from tests.adapters.conftest import format_messages_and_lm_kwargs def test_xml_adapter_format_and_parse_basic(): @@ -307,12 +308,15 @@ class StringSignature(dspy.Signature): question: str = dspy.InputField() answer: str = dspy.OutputField() - messages = dspy.XMLAdapter().format( + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), StringSignature, demos=[], inputs={"question": "why did a chicken cross the kitchen?"}, ) + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + assert messages == [ { "role": "system", @@ -349,12 +353,15 @@ class StringSignature(dspy.Signature): answer: str = dspy.InputField() judgement: str = dspy.OutputField() - messages = dspy.XMLAdapter().format( + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), StringSignature, demos=[], inputs={"question": "why did a chicken cross the kitchen?", "answer": "To get to the other side!"}, ) + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + assert messages == [ { "role": "system", @@ -400,12 +407,15 @@ class MultiAnswer(dspy.Signature): answer: str = dspy.OutputField() score: float = dspy.OutputField() - messages = dspy.XMLAdapter().format( + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), MultiAnswer, demos=[{"question": "Q1", "answer": "A1", "score": 0.9}], inputs={"question": "Q2"}, ) + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + assert messages == [ { "role": "system", @@ -454,6 +464,300 @@ class MultiAnswer(dspy.Signature): ] +def test_xml_adapter_format_exact_messages_with_history_demo_pydantic_tools_and_image(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class Location(pydantic.BaseModel): + city: str + country: str + + class Profile(pydantic.BaseModel): + name: str + location: Location + interests: list[str] + + class AnswerCard(pydantic.BaseModel): + answer: str + sources: list[str] + + class RichRenderingSignature(dspy.Signature): + """Answer using all supplied context.""" + + history: dspy.History = dspy.InputField() + image: dspy.Image = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + profile: Profile = dspy.InputField() + question: str = dspy.InputField() + answer: AnswerCard = dspy.OutputField() + + tool = dspy.Tool(search) + demo_profile = Profile( + name="Ada", + location=Location(city="London", country="UK"), + interests=["math", "machines"], + ) + current_profile = Profile( + name="Grace", + location=Location(city="Arlington", country="USA"), + interests=["compilers", "navy"], + ) + history = dspy.History( + messages=[ + { + "profile": demo_profile, + "question": "Who is Ada?", + "answer": AnswerCard(answer="Ada is a mathematician.", sources=["memory"]), + } + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), + RichRenderingSignature, + demos=[ + { + "image": dspy.Image("https://example.com/demo.png"), + "tools": [tool], + "profile": demo_profile, + "question": "What should we mention?", + "answer": AnswerCard(answer="Mention analytical engines.", sources=["demo"]), + } + ], + inputs={ + "history": history, + "image": dspy.Image("https://example.com/current.png"), + "tools": [tool], + "profile": current_profile, + "question": "What should the answer include?", + }, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `history` (History): \n' + '2. `image` (Image): \n' + '3. `tools` (list[Tool]): \n' + '4. `profile` (Profile): \n' + '5. `question` (str):\n' + 'Your output fields are:\n' + '1. `answer` (AnswerCard):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '\n' + '{history}\n' + '\n' + '\n' + '\n' + '{image}\n' + '\n' + '\n' + '\n' + '{tools}\n' + '\n' + '\n' + '\n' + '{profile}\n' + '\n' + '\n' + '\n' + '{question}\n' + '\n' + '\n' + '\n' + '{answer} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "properties": {"answer": {"type": "string", "title": "Answer"}, ' + '"sources": {"type": "array", "items": {"type": "string"}, "title": "Sources"}}, ' + '"required": ["answer", "sources"], "title": "AnswerCard"}\n' + '\n' + 'In adhering to this structure, your objective is: \n' + ' Answer using all supplied context.'}, + {"role": "user", + "content": [{"type": "text", + "text": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, + {"type": "text", + "text": '\n' + '\n' + '\n' + '\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '\n' + '\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' + '"interests": ["math", "machines"]}\n' + '\n' + '\n' + '\n' + 'What should we mention?\n' + ''}]}, + {"role": "assistant", + "content": '\n{"answer": "Mention analytical engines.", "sources": ["demo"]}\n'}, + {"role": "user", + "content": '\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, "interests": ' + '["math", "machines"]}\n' + '\n' + '\n' + '\n' + 'Who is Ada?\n' + ''}, + {"role": "assistant", + "content": '\n{"answer": "Ada is a mathematician.", "sources": ["memory"]}\n'}, + {"role": "user", + "content": [{"type": "text", "text": "\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, + {"type": "text", + "text": '\n' + '\n' + '\n' + '\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '\n' + '\n' + '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' + '"interests": ["compilers", "navy"]}\n' + '\n' + '\n' + '\n' + 'What should the answer include?\n' + '\n' + '\n' + 'Respond with the corresponding output fields wrapped in XML tags ' + '``.'}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_xml_adapter_format_exact_messages_with_nested_pydantic_output(): + class XmlAddress(pydantic.BaseModel): + city: str + country: str + + class XmlSummary(pydantic.BaseModel): + title: str + address: XmlAddress + + class PydanticSignature(dspy.Signature): + question: str = dspy.InputField() + summary: XmlSummary = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), PydanticSignature, [], {"question": "Summarize"}) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `summary` (XmlSummary):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '\n' + '{question}\n' + '\n' + '\n' + '\n' + '{summary} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "$defs": {"XmlAddress": {"type": "object", "properties": {"city": ' + '{"type": "string", "title": "City"}, "country": {"type": "string", "title": ' + '"Country"}}, "required": ["city", "country"], "title": "XmlAddress"}}, "properties": ' + '{"address": {"$ref": "#/$defs/XmlAddress"}, "title": {"type": "string", "title": ' + '"Title"}}, "required": ["title", "address"], "title": "XmlSummary"}\n' + '\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `summary`.'}, + {"role": "user", + "content": "\n" + "Summarize\n" + "\n" + "\n" + "Respond with the corresponding output fields wrapped in XML tags ``."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_xml_adapter_format_exact_messages_with_incomplete_demo(): + class IncompleteDemoSignature(dspy.Signature): + question: str = dspy.InputField() + context: str = dspy.InputField() + answer: str = dspy.OutputField() + score: float = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), + IncompleteDemoSignature, + [{"question": "Q1", "answer": "A1"}], + {"question": "Q2", "context": "C2"}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str): \n" + "2. `context` (str):\n" + "Your output fields are:\n" + "1. `answer` (str): \n" + "2. `score` (float):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "\n" + "{question}\n" + "\n" + "\n" + "\n" + "{context}\n" + "\n" + "\n" + "\n" + "{answer}\n" + "\n" + "\n" + "\n" + "{score} # note: the value you produce must be a single float value\n" + "\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `context`, produce the fields `answer`, " + "`score`."}, + {"role": "user", + "content": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "\n" + "Q1\n" + ""}, + {"role": "assistant", + "content": "\n" + "A1\n" + "\n" + "\n" + "\n" + "Not supplied for this particular example. \n" + ""}, + {"role": "user", + "content": "\n" + "Q2\n" + "\n" + "\n" + "\n" + "C2\n" + "\n" + "\n" + "Respond with the corresponding output fields wrapped in XML tags ``, then " + "``."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + def test_format_system_message(): class MySignature(dspy.Signature): """Answer the question with multiple answers and scores""" @@ -468,7 +772,7 @@ class MySignature(dspy.Signature): expected_system_message = """Your input fields are: 1. `question` (str): Your output fields are: -1. `answers` (list[str]): +1. `answers` (list[str]):\x20 2. `scores` (list[float]): All interactions will be structured in the following way, with the appropriate values filled in. @@ -483,6 +787,6 @@ class MySignature(dspy.Signature): {scores} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "number"}} -In adhering to this structure, your objective is: +In adhering to this structure, your objective is:\x20 Answer the question with multiple answers and scores""" assert system_message == expected_system_message From c827fed57da9be78d1d62dffa553f9102d8892d7 Mon Sep 17 00:00:00 2001 From: Maxime Rivest Date: Fri, 22 May 2026 07:01:13 -0400 Subject: [PATCH 06/23] Add core language model types (#9786) * Add core language model types * Fix core LM type edge cases --------- Co-authored-by: isaacbmiller --- dspy/core/types.py | 1992 ++++++++++++++++++++++++++++++++++++++ tests/core/test_types.py | 316 ++++++ 2 files changed, 2308 insertions(+) create mode 100644 dspy/core/types.py create mode 100644 tests/core/test_types.py diff --git a/dspy/core/types.py b/dspy/core/types.py new file mode 100644 index 0000000000..06bd907db4 --- /dev/null +++ b/dspy/core/types.py @@ -0,0 +1,1992 @@ +"""Normalized request, response, and stream types for DSPy language models.""" + +from __future__ import annotations + +import json +import mimetypes +from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from dataclasses import dataclass +from dataclasses import field as dataclass_field +from pprint import pformat +from typing import Annotated, Any, Literal +from urllib.parse import urlparse + +import pydantic +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class LMBasePart(BaseModel): + """A single content item in an LM message or output.""" + + type: str + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class LMSourcePart(LMBasePart): + """Content from exactly one provider-addressable source.""" + + media_type: str + data: str | None = None + url: str | None = None + file_id: str | None = None + path: str | None = None + + @model_validator(mode="after") + def validate_one_source(self) -> LMSourcePart: + _validate_one_source(self) + return self + + +class LMTextPart(LMBasePart): + """Text content.""" + + type: Literal["text"] = "text" + text: str + + +class LMImagePart(LMSourcePart): + """Image content from data, a URL, a file ID, or a local path.""" + + type: Literal["image"] = "image" + media_type: str = "image/png" + detail: Literal["low", "high", "auto"] | None = None + + +class LMAudioPart(LMSourcePart): + """Audio content from data, a URL, a file ID, or a local path.""" + + type: Literal["audio"] = "audio" + media_type: str = "audio/wav" + + +class LMVideoPart(LMSourcePart): + """Video content from data, a URL, a file ID, or a local path.""" + + type: Literal["video"] = "video" + media_type: str = "video/mp4" + + +class LMDocumentPart(LMBasePart): + """Semantic source/document content, optionally citation-enabled. + + Documents are source material: text, PDFs, reports, contracts, or other + provider-addressable evidence that benefits from title/context/citation + semantics. Use `LMBinaryPart` for opaque attachments or arbitrary bytes. + """ + + type: Literal["document"] = "document" + media_type: str = "application/pdf" + data: str | None = None + url: str | None = None + file_id: str | None = None + path: str | None = None + source: dict[str, Any] | None = None + citations: dict[str, Any] = Field(default_factory=dict) + title: str | None = None + context: str | None = None + + @model_validator(mode="after") + def validate_source(self) -> LMDocumentPart: + has_media_source = any(value is not None for value in (self.data, self.url, self.file_id, self.path)) + if self.source is not None and has_media_source: + raise ValueError("LMDocumentPart accepts either source or one of data, url, file_id, or path, not both.") + if self.source is None: + _validate_one_source(self) + elif not self.source: + raise ValueError("LMDocumentPart.source must be non-empty when provided.") + return self + + +class LMBinaryPart(LMSourcePart): + """Opaque binary content from data, a URL, a file ID, or a local path.""" + + type: Literal["binary"] = "binary" + media_type: str = "application/octet-stream" + filename: str | None = None + + +class LMToolCallPart(LMBasePart): + """A model request to call a tool. + + Use `dspy.ToolCall(...)` as a shorter public alias when constructing + assistant messages by hand. + + Args: + id: Provider call ID, when the backend uses one. + name: Name of the tool to call. + args: JSON-like arguments for the tool. + provider_data: Raw provider fields to keep with the tool call. + + Examples: + ```python + import dspy + + assistant = dspy.Assistant( + dspy.ToolCall( + id="call_1", + name="search", + args={"query": "DSPy"}, + ) + ) + ``` + """ + + type: Literal["tool_call"] = "tool_call" + id: str | None = None + name: str + args: dict[str, Any] = Field(default_factory=dict) + provider_data: dict[str, Any] = Field(default_factory=dict) + + +class LMToolResultPart(LMBasePart): + """A tool execution result sent back to a model.""" + + type: Literal["tool_result"] = "tool_result" + call_id: str | None = None + name: str | None = None + content: list[LMPart] = Field(default_factory=list) + is_error: bool = False + provider_data: dict[str, Any] = Field(default_factory=dict) + + @model_validator(mode="before") + @classmethod + def normalize_content(cls, data: Any) -> Any: + if isinstance(data, dict) and "content" in data: + data = dict(data) + content = data["content"] + if content is None: + data["content"] = [] + elif isinstance(content, list): + data["content"] = [_coerce_part(item) for item in content] + else: + data["content"] = [_coerce_part(content)] + return data + + +class LMThinkingPart(LMBasePart): + """Reasoning or thinking content returned by a model.""" + + type: Literal["thinking"] = "thinking" + text: str + redacted: bool = False + + +class LMCitationPart(LMBasePart): + """A source citation returned by a model.""" + + type: Literal["citation"] = "citation" + text: str | None = None + title: str | None = None + url: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_has_content(self) -> LMCitationPart: + if self.text is None and self.title is None and self.url is None: + raise ValueError("LMCitationPart requires at least one of text, title, or url.") + return self + + +class LMRefusalPart(LMBasePart): + """A model refusal.""" + + type: Literal["refusal"] = "refusal" + text: str + + +LMPart = Annotated[ + LMTextPart + | LMImagePart + | LMAudioPart + | LMVideoPart + | LMDocumentPart + | LMBinaryPart + | LMToolCallPart + | LMToolResultPart + | LMThinkingPart + | LMCitationPart + | LMRefusalPart, + Field(discriminator="type"), +] + + +class LMMessage(BaseModel): + """A role-attributed sequence of LM parts.""" + + role: str + parts: list[LMPart] + name: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="before") + @classmethod + def normalize_parts(cls, data: Any) -> Any: + if isinstance(data, cls): + return data + if isinstance(data, dict): + data = dict(data) + if data.get("role") == "tool" and "parts" not in data: + content = data.pop("content", None) + call_id = data.pop("tool_call_id", None) + name = data.pop("name", None) + data["parts"] = [ + LMToolResultPart( + call_id=call_id, + name=name, + content=_parts_from_openai_content(content), + ) + ] + elif "parts" not in data: + parts = _parts_from_openai_content(data.pop("content", None)) if "content" in data else [] + if "tool_calls" in data: + parts.extend(_tool_calls_from_openai(data.pop("tool_calls") or [])) + data["parts"] = parts + else: + data["parts"] = [_coerce_part(part) for part in data["parts"]] + if "tool_calls" in data: + data["parts"].extend(_tool_calls_from_openai(data.pop("tool_calls") or [])) + return data + + @property + def text(self) -> str | None: + texts = [part.text for part in self.parts if isinstance(part, LMTextPart)] + return "".join(texts) if texts else None + + +class LMToolSpec(BaseModel): + """Provider-independent schema for a tool available to an LM.""" + + type: Literal["function"] = "function" + name: str + description: str | None = None + parameters: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + provider_data: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class LMReasoningConfig(BaseModel): + """Reasoning controls for models with native reasoning support.""" + + effort: str | None = None + max_tokens: int | None = None + summary: str | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_value(cls, value: Any = None, **overrides: Any) -> LMReasoningConfig: + data = _config_data(value, str_field="effort") + if "reasoning_effort" in data and "effort" not in data: + data["effort"] = data.pop("reasoning_effort") + data.update({key: value for key, value in overrides.items() if value is not _MISSING}) + return cls(**data) + + +class LMToolChoice(BaseModel): + """Tool-choice controls for native tool-capable models.""" + + mode: Literal["auto", "required", "none"] = "auto" + # Tool names that are allowed for a model request. + allowed: list[str] | None = None + parallel: bool | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_value(cls, value: Any = None, **overrides: Any) -> LMToolChoice: + data = _config_data(value, str_field="mode") + if "parallel_tool_calls" in data and "parallel" not in data: + data["parallel"] = data.pop("parallel_tool_calls") + data.update({key: value for key, value in overrides.items() if value is not _MISSING}) + return cls(**data) + + +class LMCacheConfig(BaseModel): + """DSPy memoization cache controls for a normalized LM request. + + This cache skips the provider call entirely when DSPy finds an exact + request match. Use `LMPromptCacheConfig` for provider-side prompt/token + caching that still sends the request to the provider. + """ + + enabled: bool | None = None + rollout_id: int | str | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_value(cls, value: Any = None, **overrides: Any) -> LMCacheConfig: + data = _config_data(value, bool_field="enabled") + data.update({key: value for key, value in overrides.items() if value is not _MISSING}) + return cls(**data) + + +class LMPromptCacheConfig(BaseModel): + """Provider-side prompt/token cache controls. + + Prompt caching is not DSPy memoization. The provider call still happens, + but the backend may reuse cached prompt prefixes or KV state for lower + latency or lower input-token cost. + """ + + enabled: bool | None = None + key: str | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_value(cls, value: Any = None, **overrides: Any) -> LMPromptCacheConfig: + data = _config_data(value, bool_field="enabled") + if "prompt_cache_key" in data and "key" not in data: + data["key"] = data.pop("prompt_cache_key") + data.update({key: value for key, value in overrides.items() if value is not _MISSING}) + return cls(**data) + + +_MISSING = object() + + +def _config_data(value: Any, *, str_field: str | None = None, bool_field: str | None = None) -> dict[str, Any]: + if value is None: + return {} + if isinstance(value, BaseModel): + return value.model_dump(exclude_none=True) + if isinstance(value, Mapping): + return dict(value) + if str_field is not None and isinstance(value, str): + return {str_field: value} + if bool_field is not None and isinstance(value, bool): + return {bool_field: value} + raise TypeError(f"Cannot convert {type(value)!r} to a config object.") + + +# Keep existing `dspy.LM(...)` / `lm(...)` keyword aliases in the call compatibility path. +_KNOWN_CONFIG_KEYS = { + "temperature", + "max_tokens", + "top_p", + "stop", + "n", + "logprobs", + "response_format", + "reasoning", + "reasoning_effort", + "tool_choice", + "parallel_tool_calls", + "cache", + "rollout_id", + "prompt_cache", + "prompt_cache_key", + "extensions", +} + + +def _split_config_kwargs(kwargs: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + data: dict[str, Any] = {} + extensions = dict(kwargs.get("extensions", {}) or {}) + + for key, value in kwargs.items(): + if key == "extensions": + continue + if key in _KNOWN_CONFIG_KEYS: + data[key] = value + else: + extensions[key] = value + return data, extensions + + +def _normalize_reasoning_config(data: dict[str, Any]) -> None: + if "reasoning_effort" in data: + effort = data.pop("reasoning_effort") + if data.get("reasoning") is not None or effort is not None: + data["reasoning"] = LMReasoningConfig.from_value(data.get("reasoning"), effort=effort) + elif "reasoning" in data and data["reasoning"] is not None: + data["reasoning"] = LMReasoningConfig.from_value(data["reasoning"]) + + +def _normalize_tool_choice_config(data: dict[str, Any]) -> None: + parallel = data.pop("parallel_tool_calls", _MISSING) + if data.get("tool_choice") is not None or parallel not in (_MISSING, None): + data["tool_choice"] = LMToolChoice.from_value(data.get("tool_choice"), parallel=parallel) + + +def _normalize_cache_config(data: dict[str, Any]) -> None: + rollout_id = data.pop("rollout_id", _MISSING) + if data.get("cache") is not None or rollout_id not in (_MISSING, None): + data["cache"] = LMCacheConfig.from_value(data.get("cache"), rollout_id=rollout_id) + + +def _normalize_prompt_cache_config(data: dict[str, Any]) -> None: + prompt_cache_key = data.pop("prompt_cache_key", _MISSING) + if data.get("prompt_cache") is not None or prompt_cache_key not in (_MISSING, None): + data["prompt_cache"] = LMPromptCacheConfig.from_value(data.get("prompt_cache"), key=prompt_cache_key) + + +class LMConfig(BaseModel): + """Common generation controls for an LM request.""" + + temperature: float | None = None + max_tokens: int | None = None + top_p: float | None = None + stop: list[str] | None = None + n: int | None = None + logprobs: bool | int | None = None + response_format: Any | None = None + reasoning: LMReasoningConfig | None = None + tool_choice: LMToolChoice | None = None + cache: LMCacheConfig | None = None + prompt_cache: LMPromptCacheConfig | None = None + extensions: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + @classmethod + def from_kwargs(cls, **kwargs: Any) -> LMConfig: + data, extensions = _split_config_kwargs(kwargs) + _normalize_reasoning_config(data) + _normalize_tool_choice_config(data) + _normalize_cache_config(data) + _normalize_prompt_cache_config(data) + data["extensions"] = extensions + return cls(**data) + + +def _merge_lm_config(left: LMConfig | None, right: LMConfig | None) -> LMConfig | None: + if left is None: + return right + if right is None: + return left + + data = left.model_dump() + right_data = right.model_dump(exclude_none=True) + for key in ("reasoning", "tool_choice", "cache", "prompt_cache"): + if key in right_data and isinstance(data.get(key), dict) and isinstance(right_data[key], dict): + right_data[key] = {**data[key], **right_data[key]} + extensions = {**left.extensions, **right.extensions} + data.update(right_data) + data["extensions"] = extensions + return LMConfig(**data) + + +def _merge_config_overrides(config: LMConfig, kwargs: dict[str, Any]) -> LMConfig: + data = config.model_dump() + extensions = dict(config.extensions) + + direct_keys = { + "temperature", + "max_tokens", + "top_p", + "stop", + "n", + "logprobs", + "response_format", + } + for key in direct_keys & kwargs.keys(): + data[key] = kwargs[key] + + if "reasoning" in kwargs or "reasoning_effort" in kwargs: + data["reasoning"] = LMReasoningConfig.from_value( + kwargs.get("reasoning", data.get("reasoning")), + effort=kwargs.get("reasoning_effort", _MISSING), + ) + + if "tool_choice" in kwargs or "parallel_tool_calls" in kwargs: + data["tool_choice"] = LMToolChoice.from_value( + kwargs.get("tool_choice", data.get("tool_choice")), + parallel=kwargs.get("parallel_tool_calls", _MISSING), + ) + + if "cache" in kwargs or "rollout_id" in kwargs: + data["cache"] = LMCacheConfig.from_value( + kwargs.get("cache", data.get("cache")), + rollout_id=kwargs.get("rollout_id", _MISSING), + ) + + if "prompt_cache" in kwargs or "prompt_cache_key" in kwargs: + data["prompt_cache"] = LMPromptCacheConfig.from_value( + kwargs.get("prompt_cache", data.get("prompt_cache")), + key=kwargs.get("prompt_cache_key", _MISSING), + ) + + if "extensions" in kwargs: + extra = kwargs["extensions"] + if extra is None: + extensions = {} + elif isinstance(extra, Mapping): + extensions.update(extra) + else: + raise TypeError("`extensions` override must be a mapping or None.") + + handled = _KNOWN_CONFIG_KEYS + for key, value in kwargs.items(): + if key not in handled: + extensions[key] = value + data["extensions"] = extensions + return LMConfig(**data) + + +@dataclass +class LMRequestPatch: + """A partial normalized LM request contributed while rendering a DSPy call. + + `LMRequest` is the complete object a `LanguageModel` receives. A patch is + the smaller, composable unit that DSPy type strategies can contribute while + an adapter is still building that request: extra messages, extra parts, + native tools, native config, or signature fields that should be hidden from + the outer adapter's ordinary text/JSON/XML rendering. + """ + + messages: list[LMMessage] = dataclass_field(default_factory=list) + system_parts: list[LMPart] = dataclass_field(default_factory=list) + user_parts: list[LMPart] = dataclass_field(default_factory=list) + assistant_parts: list[LMPart] = dataclass_field(default_factory=list) + tools: list[LMToolSpec] = dataclass_field(default_factory=list) + config: LMConfig | None = None + delete_input_fields: tuple[str, ...] = () + delete_output_fields: tuple[str, ...] = () + metadata: dict[str, Any] = dataclass_field(default_factory=dict) + + def merge(self, other: LMRequestPatch) -> LMRequestPatch: + """Return a new patch containing this patch followed by `other`.""" + return LMRequestPatch( + messages=[*self.messages, *other.messages], + system_parts=[*self.system_parts, *other.system_parts], + user_parts=[*self.user_parts, *other.user_parts], + assistant_parts=[*self.assistant_parts, *other.assistant_parts], + tools=[*self.tools, *other.tools], + config=_merge_lm_config(self.config, other.config), + delete_input_fields=(*self.delete_input_fields, *other.delete_input_fields), + delete_output_fields=(*self.delete_output_fields, *other.delete_output_fields), + metadata={**self.metadata, **other.metadata}, + ) + + def as_lm_kwargs(self) -> dict[str, Any]: + """Return the legacy kwargs implied by this patch. + + This keeps the first implementation usable with today's adapter call + path, which still passes `lm_kwargs` rather than an `LMRequestPatch` all + the way down. Message and part patches are intentionally not flattened + here; they require the next adapter-call refactor. + """ + kwargs = self.config.model_dump(exclude_none=True) if self.config is not None else {} + extensions = kwargs.pop("extensions", {}) or {} + kwargs = {**extensions, **kwargs} + if self.tools: + kwargs["tools"] = list(self.tools) + return kwargs + + +class LMRequest(BaseModel): + """A normalized request passed to a `LanguageModel`.""" + + model: str + messages: list[LMMessage] + tools: list[LMToolSpec] = Field(default_factory=list) + config: LMConfig = Field(default_factory=LMConfig) + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + @classmethod + def from_call( + cls, + *, + model: str, + items: tuple[Any, ...] = (), + prompt: str | None = None, + messages: list[dict[str, Any] | LMMessage] | None = None, + tools: list[Any] | None = None, + **kwargs: Any, + ) -> LMRequest: + if messages is not None and (items or prompt is not None): + raise ValueError("Pass messages or direct-call inputs, not both.") + + collected_tools: list[Any] = list(tools or []) + if messages is not None: + normalized_messages = [_coerce_message(message) for message in messages] + else: + normalized_messages, positional_tools = _messages_from_items(items, prompt=prompt) + collected_tools.extend(positional_tools) + + config = LMConfig.from_kwargs(**kwargs) + return cls( + model=model, + messages=normalized_messages, + tools=[_coerce_tool_spec(tool) for tool in collected_tools], + config=config, + ) + + @classmethod + def from_prompt_or_messages( + cls, + *, + model: str, + prompt: str | None = None, + messages: list[dict[str, Any] | LMMessage] | None = None, + **kwargs: Any, + ) -> LMRequest: + return cls.from_call(model=model, prompt=prompt, messages=messages, **kwargs) + + def with_config_overrides(self, **kwargs: Any) -> LMRequest: + """Return a copy with explicit request config overrides applied. + + Only fields implied by the supplied keyword arguments are changed. This + preserves existing grouped config such as `cache`, `prompt_cache`, + `tool_choice`, `reasoning`, `stop`, and provider-specific + `extensions` when an unrelated setting is overridden. + """ + if not kwargs: + return self + merged = _merge_config_overrides(self.config, kwargs) + return self.model_copy(update={"config": merged}, deep=True) + + +class LMUsage(BaseModel): + """Token and timing usage for one LM request. + + Both DSPy token names and provider token names are populated because both + are existing user-visible history/usage interfaces. + """ + + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + prompt_tokens: int | None = None + completion_tokens: int | None = None + reasoning_tokens: int | None = None + cache_read_tokens: int | None = None + cache_write_tokens: int | None = None + input_audio_tokens: int | None = None + output_audio_tokens: int | None = None + details: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="allow") + + @model_validator(mode="after") + def fill_aliases(self) -> LMUsage: + if self.input_tokens is None and self.prompt_tokens is not None: + self.input_tokens = self.prompt_tokens + if self.output_tokens is None and self.completion_tokens is not None: + self.output_tokens = self.completion_tokens + if self.prompt_tokens is None and self.input_tokens is not None: + self.prompt_tokens = self.input_tokens + if self.completion_tokens is None and self.output_tokens is not None: + self.completion_tokens = self.output_tokens + if self.total_tokens is None and self.input_tokens is not None and self.output_tokens is not None: + self.total_tokens = self.input_tokens + self.output_tokens + return self + + +class LMOutput(BaseModel): + """One generated candidate in an LM response.""" + + parts: list[LMPart] = Field(default_factory=list) + finish_reason: str | None = None + truncated: bool = False + logprobs: Any | None = None + provider_output: Any | None = Field(default=None, exclude=True) + provider_data: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + @model_validator(mode="before") + @classmethod + def normalize_parts(cls, data: Any) -> Any: + if isinstance(data, dict) and "parts" in data: + data = dict(data) + data["parts"] = [_coerce_part(part) for part in data["parts"]] + return data + + @property + def text(self) -> str | None: + texts = [part.text for part in self.parts if isinstance(part, LMTextPart)] + return "".join(texts) if texts else None + + @property + def reasoning_content(self) -> str | None: + texts = [part.text for part in self.parts if isinstance(part, LMThinkingPart)] + return "".join(texts) if texts else None + + @property + def tool_calls(self) -> list[LMToolCallPart]: + return [part for part in self.parts if isinstance(part, LMToolCallPart)] + + @property + def citations(self) -> list[LMCitationPart]: + return [part for part in self.parts if isinstance(part, LMCitationPart)] + + @property + def images(self) -> list[LMImagePart]: + return [part for part in self.parts if isinstance(part, LMImagePart)] + + @property + def audio(self) -> list[LMAudioPart]: + return [part for part in self.parts if isinstance(part, LMAudioPart)] + + @property + def videos(self) -> list[LMVideoPart]: + return [part for part in self.parts if isinstance(part, LMVideoPart)] + + @property + def documents(self) -> list[LMDocumentPart]: + return [part for part in self.parts if isinstance(part, LMDocumentPart)] + + @property + def binaries(self) -> list[LMBinaryPart]: + return [part for part in self.parts if isinstance(part, LMBinaryPart)] + + @property + def refusal(self) -> str | None: + refusals = [part.text for part in self.parts if isinstance(part, LMRefusalPart)] + return "".join(refusals) if refusals else None + + def to_value(self) -> Any: + values = [_part_to_value(part) for part in self.parts] + values = [value for value in values if value is not None] + if len(values) == 1 and isinstance(values[0], str) and self.logprobs is None: + return values[0] + return values + + def to_output_dict(self) -> dict[str, Any]: + data: dict[str, Any] = {"text": self.text} + if self.reasoning_content is not None: + data["reasoning_content"] = self.reasoning_content + if self.tool_calls: + data["tool_calls"] = [_tool_call_to_provider_dict(call) for call in self.tool_calls] + if self.citations: + data["citations"] = [citation.model_dump(exclude_none=True) for citation in self.citations] + if self.logprobs is not None: + data["logprobs"] = self.logprobs + return data + + +class LMResponse(BaseModel): + """The normalized result of one LM request.""" + + model: str | None = None + outputs: list[LMOutput] = Field(min_length=1) + usage: LMUsage | dict[str, Any] | None = None + cost: float | None = None + cache_hit: bool = False + response_id: str | None = None + provider_response: Any | None = Field(default=None, exclude=True) + provider_data: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + @model_validator(mode="before") + @classmethod + def normalize_usage(cls, data: Any) -> Any: + if isinstance(data, dict) and isinstance(data.get("usage"), dict): + data = dict(data) + data["usage"] = LMUsage(**data["usage"]) + return data + + @classmethod + def from_text( + cls, + text: str, + *, + model: str | None = None, + usage: LMUsage | dict[str, Any] | None = None, + cost: float | None = None, + cache_hit: bool = False, + **kwargs: Any, + ) -> LMResponse: + return cls( + model=model, + outputs=[LMOutput(parts=[LMTextPart(text=text)])], + usage=usage, + cost=cost, + cache_hit=cache_hit, + **kwargs, + ) + + def __iter__(self): + return iter(self.to_values()) + + def __getitem__(self, index: int) -> Any: + return self.to_values()[index] + + def __len__(self) -> int: + return len(self.outputs) + + @property + def output(self) -> LMOutput: + return self.outputs[0] + + @property + def parts(self) -> list[LMPart]: + return self.output.parts + + @property + def text(self) -> str | None: + return self.output.text + + @property + def reasoning_content(self) -> str | None: + return self.output.reasoning_content + + @property + def tool_calls(self) -> list[LMToolCallPart]: + return self.output.tool_calls + + @property + def citations(self) -> list[LMCitationPart]: + return self.output.citations + + @property + def images(self) -> list[LMImagePart]: + return self.output.images + + @property + def audio(self) -> list[LMAudioPart]: + return self.output.audio + + @property + def videos(self) -> list[LMVideoPart]: + return self.output.videos + + @property + def documents(self) -> list[LMDocumentPart]: + return self.output.documents + + @property + def binaries(self) -> list[LMBinaryPart]: + return self.output.binaries + + def to_values(self) -> list[Any]: + return [output.to_value() for output in self.outputs] + + def to_outputs(self) -> list[Any]: + outputs: list[Any] = [] + for output in self.outputs: + if _requires_output_dict(output): + outputs.append(output.to_output_dict()) + else: + outputs.append(output.to_value()) + return outputs + + def to_legacy_outputs(self) -> list[Any]: + return self.to_outputs() + + def usage_as_dict(self) -> dict[str, Any]: + if self.usage is None: + return {} + if isinstance(self.usage, LMUsage): + return self.usage.model_dump(exclude_none=True) + return dict(self.usage) + + +class LMHistoryEntry(BaseModel, Mapping[str, Any]): + """A typed history record that can be read like a dictionary. + + Store the canonical request and response, then derive legacy convenience + fields such as `outputs`, `usage`, `messages`, and `kwargs` on demand. + Because this class implements `Mapping`, existing history code can keep + using `entry["messages"]`, `entry.get("prompt")`, `entry.items()`, and + `dict(entry)`. + """ + + request: LMRequest + response: LMResponse + timestamp: str + uuid: str + model_type: str | None = None + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") + + @model_validator(mode="before") + @classmethod + def drop_derived_fields(cls, data: Any) -> Any: + if isinstance(data, dict): + data = dict(data) + for key in _HISTORY_DERIVED_KEYS: + data.pop(key, None) + return data + + @property + def outputs(self) -> list[Any]: + return self.response.to_outputs() + + @property + def usage(self) -> dict[str, Any]: + return self.response.usage_as_dict() + + @property + def cost(self) -> float | None: + return self.response.cost + + @property + def model(self) -> str: + return self.request.model + + @property + def prompt(self) -> str | None: + return _history_request_prompt(self.request) + + @property + def messages(self) -> list[dict[str, Any]] | None: + return _history_request_messages_as_openai(self.request) + + @property + def kwargs(self) -> dict[str, Any]: + return _history_request_kwargs(self.request) + + @property + def response_model(self) -> str | None: + return self.response.model + + def __getitem__(self, key: str) -> Any: + return self._mapping()[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._mapping()) + + def __len__(self) -> int: + return len(self._mapping()) + + def __repr__(self) -> str: + formatted = pformat(self._essential_mapping(), width=100, sort_dicts=False) + return f"LMHistoryEntry(\n{formatted}\n)" + + def __str__(self) -> str: + return repr(self) + + def to_dict(self, *, mode: str = "python", exclude_none: bool = False, **kwargs: Any) -> dict[str, Any]: + """Return this history entry as a plain dictionary.""" + if kwargs: + return self.model_dump(mode=mode, exclude_none=exclude_none, **kwargs) + data = self._mapping() + if mode != "python": + data = json.loads(json.dumps(data, default=_json_default)) + if exclude_none: + data = {key: value for key, value in data.items() if value is not None} + return data + + def _essential_mapping(self) -> dict[str, Any]: + data = self.model_dump(mode="python", exclude_none=True) + data.update(self.model_extra or {}) + return data + + def _mapping(self) -> dict[str, Any]: + data = self._essential_mapping() + data.update({key: getattr(self, key) for key in _HISTORY_DERIVED_KEYS}) + return {key: value for key, value in data.items() if value is not None} + + +_HISTORY_DERIVED_KEYS = ( + "outputs", + "usage", + "cost", + "model", + "prompt", + "messages", + "kwargs", + "response_model", +) + + +def _json_default(value: Any) -> Any: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json", exclude_none=True) + return str(value) + + +class LMDelta(BaseModel): + """Base class for streamed content deltas.""" + + type: str + + +class LMTextDelta(LMDelta): + type: Literal["text_delta"] = "text_delta" + text: str + + +class LMThinkingDelta(LMDelta): + type: Literal["thinking_delta"] = "thinking_delta" + text: str + + +class LMToolCallDelta(LMDelta): + type: Literal["tool_call_delta"] = "tool_call_delta" + id: str | None = None + name: str | None = None + args_delta: str | None = None + + +class LMCitationDelta(LMDelta): + type: Literal["citation_delta"] = "citation_delta" + citation: LMCitationPart + + +class LMImageDelta(LMDelta): + type: Literal["image_delta"] = "image_delta" + image: LMImagePart + + +class LMAudioDelta(LMDelta): + type: Literal["audio_delta"] = "audio_delta" + audio: LMAudioPart + + +LMAnyDelta = Annotated[ + LMTextDelta | LMThinkingDelta | LMToolCallDelta | LMCitationDelta | LMImageDelta | LMAudioDelta, + Field(discriminator="type"), +] + + +class LMStreamEvent(BaseModel): + """Base class for normalized LM stream events.""" + + type: str + + +class LMStreamStartEvent(LMStreamEvent): + type: Literal["start"] = "start" + model: str | None = None + + +class LMStreamDeltaEvent(LMStreamEvent): + type: Literal["delta"] = "delta" + output_index: int = Field(default=0, ge=0) + part_index: int = Field(ge=0) + delta: LMAnyDelta + + +class LMStreamOutputEndEvent(LMStreamEvent): + type: Literal["output_end"] = "output_end" + output_index: int = Field(default=0, ge=0) + finish_reason: str | None = None + truncated: bool = False + + +class LMStreamEndEvent(LMStreamEvent): + type: Literal["end"] = "end" + usage: LMUsage | dict[str, Any] | None = None + cost: float | None = None + response: LMResponse | None = None + + +class LMStreamErrorEvent(LMStreamEvent): + type: Literal["error"] = "error" + error: Exception + + model_config = ConfigDict(arbitrary_types_allowed=True) + + +class LMOutputBuilder: + """Assemble streamed LM events into a final `LMResponse`.""" + + def __init__(self): + self.model: str | None = None + self._parts: dict[int, list[LMPart | None]] = {} + self._finish_reasons: dict[int, str | None] = {} + self._truncated: dict[int, bool] = {} + + def apply(self, event: LMStreamEvent) -> LMResponse | None: + if isinstance(event, LMStreamStartEvent): + self.model = event.model + return None + if isinstance(event, LMStreamDeltaEvent): + self._apply_delta(event) + return None + if isinstance(event, LMStreamOutputEndEvent): + self._finish_reasons[event.output_index] = event.finish_reason + self._truncated[event.output_index] = event.truncated + return None + if isinstance(event, LMStreamEndEvent): + if event.response is not None: + return event.response + return self.to_response(usage=event.usage, cost=event.cost) + if isinstance(event, LMStreamErrorEvent): + raise event.error + return None + + def to_response(self, *, usage: LMUsage | dict[str, Any] | None = None, cost: float | None = None) -> LMResponse: + output_indices = set(self._parts) | set(self._finish_reasons) | set(self._truncated) + if not output_indices: + output_indices = {0} + max_index = max(output_indices) + expected_indices = set(range(max_index + 1)) + if output_indices != expected_indices: + missing = sorted(expected_indices - output_indices) + raise ValueError(f"Stream output indices must be contiguous from 0; missing indices: {missing}.") + + outputs = [] + for output_index in range(max_index + 1): + part_buffer = self._parts.get(output_index, []) + missing_part_indices = [index for index, part in enumerate(part_buffer) if part is None] + if missing_part_indices: + raise ValueError( + f"Stream part indices for output {output_index} must be contiguous; " + f"missing indices: {missing_part_indices}." + ) + parts = [_finalize_stream_part(part) for part in part_buffer] + outputs.append( + LMOutput( + parts=parts, + finish_reason=self._finish_reasons.get(output_index), + truncated=self._truncated.get(output_index, False), + ) + ) + return LMResponse(model=self.model, outputs=outputs, usage=usage, cost=cost) + + def _apply_delta(self, event: LMStreamDeltaEvent) -> None: + parts = self._parts.setdefault(event.output_index, []) + while len(parts) <= event.part_index: + parts.append(None) + + current = parts[event.part_index] + delta = event.delta + if isinstance(delta, LMThinkingDelta): + if current is not None and not isinstance(current, LMThinkingPart): + raise ValueError("Cannot apply thinking delta to a non-thinking stream part.") + text = (current.text if isinstance(current, LMThinkingPart) else "") + delta.text + parts[event.part_index] = LMThinkingPart(text=text) + elif isinstance(delta, LMTextDelta): + if current is not None and not isinstance(current, LMTextPart): + raise ValueError("Cannot apply text delta to a non-text stream part.") + text = (current.text if isinstance(current, LMTextPart) else "") + delta.text + parts[event.part_index] = LMTextPart(text=text) + elif isinstance(delta, LMToolCallDelta): + if current is not None and not isinstance(current, LMToolCallPart): + raise ValueError("Cannot apply tool-call delta to a non-tool-call stream part.") + buffer = "" + if isinstance(current, LMToolCallPart): + buffer = current.provider_data.get("args_buffer", "") + buffer += delta.args_delta or "" + args = _parse_json_object(buffer) + parts[event.part_index] = LMToolCallPart( + id=delta.id if delta.id is not None else getattr(current, "id", None), + name=delta.name if delta.name is not None else getattr(current, "name", ""), + args=args, + provider_data={"args_buffer": buffer}, + ) + elif isinstance(delta, LMCitationDelta): + if current is not None and not isinstance(current, LMCitationPart): + raise ValueError("Cannot apply citation delta to a different stream part type.") + parts[event.part_index] = delta.citation + elif isinstance(delta, LMImageDelta): + if current is not None and not isinstance(current, LMImagePart): + raise ValueError("Cannot apply image delta to a different stream part type.") + parts[event.part_index] = delta.image + elif isinstance(delta, LMAudioDelta): + if current is not None and not isinstance(current, LMAudioPart): + raise ValueError("Cannot apply audio delta to a different stream part type.") + parts[event.part_index] = delta.audio + + +class LMStream: + """Synchronous LM stream with a final `LMResponse` result.""" + + def __init__( + self, + *, + request: LMRequest, + events: Iterator[LMStreamEvent], + finalize: Callable[[LMRequest, LMResponse], LMResponse], + ): + self.request = request + self._events = events + self._finalize = finalize + self._builder = LMOutputBuilder() + self._result: LMResponse | None = None + + def __iter__(self) -> Iterator[LMStreamEvent]: + for event in self._events: + response = self._builder.apply(event) + if response is not None: + self._result = self._finalize(self.request, response) + yield event + + def result(self) -> LMResponse: + if self._result is None: + raise RuntimeError("Stream has not completed yet.") + return self._result + + +class AsyncLMStream: + """Asynchronous LM stream with a final `LMResponse` result.""" + + def __init__( + self, + *, + request: LMRequest, + events: AsyncIterator[LMStreamEvent], + finalize: Callable[[LMRequest, LMResponse], LMResponse], + ): + self.request = request + self._events = events + self._finalize = finalize + self._builder = LMOutputBuilder() + self._result: LMResponse | None = None + + async def __aiter__(self) -> AsyncIterator[LMStreamEvent]: + async for event in self._events: + response = self._builder.apply(event) + if response is not None: + self._result = self._finalize(self.request, response) + yield event + + def result(self) -> LMResponse: + if self._result is None: + raise RuntimeError("Stream has not completed yet.") + return self._result + + +def System(*parts: Any, name: str | None = None, metadata: dict[str, Any] | None = None) -> LMMessage: # noqa: N802 + """Create a system message for a direct LM call. + + A system message gives model-level instructions, such as tone, scope, or + formatting rules. Pass text, media parts, or normalized `LMPart` objects; + DSPy stores them as one `LMMessage` with role `"system"`. + + Args: + *parts: Text, DSPy media objects, or normalized LM parts to include in + the message. + name: Optional sender name for providers that support named messages. + metadata: Extra information to keep with the message. + + Returns: + An `LMMessage` that can be passed to `dspy.LanguageModel`, `dspy.LM`, or + `dspy.LMRequest`. + + Examples: + System instruction with a user turn: + + ```python + import dspy + + lm = dspy.LanguageModel(model="test/model") + request = lm.normalize_request( + dspy.System("You are concise."), + dspy.User("What is DSPy?"), + ) + ``` + + See Also: + [`dspy.User`][dspy.User] + [`dspy.Assistant`][dspy.Assistant] + [`dspy.LMRequest`][dspy.LMRequest] + """ + return LMMessage(role="system", parts=[_coerce_part(part) for part in parts], name=name, metadata=metadata or {}) + + +def Developer(*parts: Any, name: str | None = None, metadata: dict[str, Any] | None = None) -> LMMessage: # noqa: N802 + """Create a developer message for a direct LM call. + + A developer message carries instructions that sit between system guidance + and user content. Use it when a provider supports a `"developer"` role and + you want to keep implementation guidance separate from the user's request. + + Args: + *parts: Text, DSPy media objects, or normalized LM parts to include in + the message. + name: Optional sender name for providers that support named messages. + metadata: Extra information to keep with the message. + + Returns: + An `LMMessage` with role `"developer"`. + + Examples: + Add house-style instructions: + + ```python + import dspy + + request = dspy.LMRequest.from_call( + model="openai/gpt-4o-mini", + items=( + dspy.System("You are a technical editor."), + dspy.Developer("Prefer short examples."), + dspy.User("Explain callbacks."), + ), + ) + ``` + + See Also: + [`dspy.System`][dspy.System] + [`dspy.User`][dspy.User] + """ + return LMMessage(role="developer", parts=[_coerce_part(part) for part in parts], name=name, metadata=metadata or {}) + + +def User(*parts: Any, name: str | None = None, metadata: dict[str, Any] | None = None) -> LMMessage: # noqa: N802 + """Create a user message for a direct LM call. + + A user message contains the request or data you want the model to answer. + Pass plain text for simple prompts, or mix text with images, audio, documents, + binary attachments, and normalized LM parts for multimodal calls. + + Args: + *parts: Text, DSPy media objects, or normalized LM parts to include in + the message. + name: Optional sender name for providers that support named messages. + metadata: Extra information to keep with the message. + + Returns: + An `LMMessage` with role `"user"`. + + Examples: + Multi-turn LM call: + + ```python + import dspy + + lm = dspy.LM("openai/gpt-4o-mini") + response = lm( + dspy.User("What is DSPy?"), + dspy.Assistant("DSPy is a framework for programming LM pipelines."), + dspy.User("Say that in five words."), + ) + ``` + + Multi-turn call with media: + + ```python + import dspy + + lm = dspy.LM("openai/gpt-4o-mini") + response = lm( + dspy.System("Answer in one sentence."), + dspy.User( + "Describe this image.", + dspy.Image("https://example.com/dog.png"), + ), + ) + ``` + + For a single user turn, pass the parts directly to `lm(...)` instead: + + ```python + response = lm("Describe this image.", dspy.Image("https://example.com/dog.png")) + ``` + + Explicit `LMRequest` for custom LM authors and advanced users: + + ```python + import dspy + + lm = dspy.LM("openai/gpt-4o-mini") + request = dspy.LMRequest( + model="openai/gpt-4o-mini", + messages=[ + dspy.System("You are concise."), + dspy.User( + "Describe this image.", + dspy.Image("https://example.com/dog.png"), + ), + ], + config=dspy.LMConfig(temperature=0.2, max_tokens=200), + ) + + response = lm(request) + ``` + + See Also: + [`dspy.System`][dspy.System] + [`dspy.Assistant`][dspy.Assistant] + [`dspy.ToolResult`][dspy.ToolResult] + """ + return LMMessage(role="user", parts=[_coerce_part(part) for part in parts], name=name, metadata=metadata or {}) + + +def Assistant(*parts: Any, name: str | None = None, metadata: dict[str, Any] | None = None) -> LMMessage: # noqa: N802 + """Create an assistant message for a direct LM call. + + An assistant message represents a previous model response. Use it when you + build a multi-turn request by hand, or when you send tool calls back to the + model before adding tool results. + + Args: + *parts: Text, tool calls, citations, media parts, or other normalized LM + parts from an assistant turn. + name: Optional sender name for providers that support named messages. + metadata: Extra information to keep with the message. + + Returns: + An `LMMessage` with role `"assistant"`. + + Examples: + Continue a conversation: + + ```python + import dspy + + request = dspy.LMRequest.from_call( + model="openai/gpt-4o-mini", + items=( + dspy.User("What is DSPy?"), + dspy.Assistant("DSPy is a framework for programming LM pipelines."), + dspy.User("Say that in five words."), + ), + ) + ``` + + See Also: + [`dspy.User`][dspy.User] + [`dspy.ToolCall`][dspy.ToolCall] + [`dspy.ToolResult`][dspy.ToolResult] + """ + return LMMessage(role="assistant", parts=[_coerce_part(part) for part in parts], name=name, metadata=metadata or {}) + + +ToolCall = LMToolCallPart +"""Create a tool-call part for an assistant message. + +`ToolCall` is an alias for `LMToolCallPart`. Use it inside `dspy.Assistant(...)` +when you want to include a model-requested tool call in a normalized +conversation. +""" + + +def ToolResult( # noqa: N802 + *parts: Any, + call_id: str | None = None, + name: str | None = None, + content: Any | None = None, + is_error: bool = False, +) -> LMMessage: + """Create a tool-result message for a direct LM call. + + A tool-result message sends the output of a tool back to the model. Pass the + returned text or media as `*parts` or with `content=...`, and include the + `call_id` from the matching assistant tool call when the provider uses call + IDs. + + Args: + *parts: Text, DSPy media objects, or normalized LM parts returned by the + tool. If you pass one `LMToolResultPart`, DSPy uses it directly. + call_id: Identifier of the assistant tool call this result answers. + name: Tool name associated with the result. + content: Optional tool output passed by keyword. Use this when adapting + OpenAI-style code that stores the result under `content`. + is_error: Whether this result represents a failed tool execution. + + Returns: + An `LMMessage` with role `"tool"` and one `LMToolResultPart`. + + Examples: + Send a weather result back to the model: + + ```python + import dspy + + messages = [ + dspy.User("What is the weather in Paris?"), + dspy.Assistant( + dspy.ToolCall( + id="call_1", + name="get_weather", + args={"location": "Paris"}, + ) + ), + dspy.ToolResult( + '{"temperature": "22", "unit": "celsius"}', + call_id="call_1", + name="get_weather", + ), + dspy.User("Summarize the result."), + ] + ``` + + See Also: + [`dspy.ToolCall`][dspy.ToolCall] + [`dspy.Assistant`][dspy.Assistant] + [`dspy.Tool`][dspy.Tool] + """ + if content is not None: + if parts: + raise TypeError("Pass tool output either as positional parts or as `content=...`, not both.") + parts = tuple(content if isinstance(content, list) else [content]) + + if len(parts) == 1 and isinstance(parts[0], LMToolResultPart): + result = parts[0] + else: + result = LMToolResultPart( + call_id=call_id, + name=name, + content=[_coerce_part(part) for part in parts], + is_error=is_error, + ) + return LMMessage(role="tool", parts=[result]) + + +def _history_request_prompt(request: LMRequest) -> str | None: + if len(request.messages) != 1: + return None + message = request.messages[0] + if message.role != "user" or len(message.parts) != 1: + return None + part = message.parts[0] + return part.text if isinstance(part, LMTextPart) else None + + +def _history_request_messages_as_openai(request: LMRequest) -> list[dict[str, Any]]: + messages = [] + for message in request.messages: + if message.role == "assistant": + tool_calls = [part for part in message.parts if isinstance(part, LMToolCallPart)] + content_parts = [part for part in message.parts if not isinstance(part, LMToolCallPart)] + item: dict[str, Any] = { + "role": "assistant", + "content": _history_message_parts_as_openai_content(content_parts) if content_parts else None, + } + if tool_calls: + item["tool_calls"] = [_history_tool_call_as_openai(call) for call in tool_calls] + elif message.role == "tool" and len(message.parts) == 1 and isinstance(message.parts[0], LMToolResultPart): + result = message.parts[0] + item = {"role": "tool", "content": _history_tool_result_content(result)} + if result.call_id is not None: + item["tool_call_id"] = result.call_id + if result.name is not None: + item["name"] = result.name + else: + item = { + "role": message.role, + "content": _history_message_parts_as_openai_content(message.parts), + } + if message.name is not None and "name" not in item: + item["name"] = message.name + messages.append(item) + return messages + + +def _history_tool_call_as_openai(call: LMToolCallPart) -> dict[str, Any]: + data: dict[str, Any] = { + "type": "function", + "function": { + "name": call.name, + "arguments": json.dumps(call.args), + }, + } + if call.id is not None: + data["id"] = call.id + return data + + +def _history_tool_result_content(result: LMToolResultPart) -> str: + chunks = [] + for part in result.content: + if isinstance(part, LMTextPart): + chunks.append(part.text) + else: + chunks.append(json.dumps(part.model_dump(mode="json", exclude_none=True), ensure_ascii=False)) + return "".join(chunks) + + +def _history_message_parts_as_openai_content(parts: list[LMPart]) -> str | list[dict[str, Any]]: + if len(parts) == 1 and isinstance(parts[0], LMTextPart): + return parts[0].text + return [_history_part_as_openai_content(part) for part in parts] + + +def _history_part_as_openai_content(part: LMPart) -> dict[str, Any]: + if isinstance(part, LMTextPart): + return {"type": "text", "text": part.text} + if isinstance(part, LMImagePart): + return {"type": "image_url", "image_url": {"url": _history_part_source(part)}} + if isinstance(part, LMAudioPart): + input_audio = {"format": _history_media_format(part.media_type)} + if part.data is not None: + if part.data.startswith("data:"): + media_type, data = _split_data_uri(part.data) + input_audio["format"] = _history_media_format(media_type) + input_audio["data"] = data + else: + input_audio["data"] = part.data + elif part.url is not None: + input_audio["url"] = part.url + elif part.file_id is not None: + input_audio["file_id"] = part.file_id + elif part.path is not None: + input_audio["path"] = part.path + return { + "type": "input_audio", + "input_audio": input_audio, + } + if isinstance(part, LMVideoPart): + video = {"media_type": part.media_type} + if part.data is not None: + video["data"] = _history_part_source(part) + elif part.url is not None: + video["url"] = part.url + elif part.file_id is not None: + video["file_id"] = part.file_id + elif part.path is not None: + video["path"] = part.path + return {"type": "video", "video": video} + if isinstance(part, LMDocumentPart): + data = {"type": "document"} + if part.source is not None: + data["source"] = part.source + else: + data["source"] = _history_part_source(part) + data["media_type"] = part.media_type + if part.citations: + data["citations"] = part.citations + if part.title is not None: + data["title"] = part.title + if part.context is not None: + data["context"] = part.context + return data + if isinstance(part, LMBinaryPart): + return { + "type": "binary", + "binary": { + key: value + for key, value in { + "data": _history_part_source(part), + "file_id": part.file_id, + "filename": part.filename, + "media_type": part.media_type, + }.items() + if value is not None + }, + } + return part.model_dump(exclude_none=True) + + +def _history_part_source(part: LMImagePart | LMAudioPart | LMVideoPart | LMDocumentPart | LMBinaryPart) -> str | None: + if part.data is not None: + return part.data if part.data.startswith("data:") else f"data:{part.media_type};base64,{part.data}" + return part.url or part.file_id or part.path + + +def _history_media_format(media_type: str) -> str: + return media_type.split("/", 1)[1] if "/" in media_type else media_type + + +def _history_request_kwargs(request: LMRequest) -> dict[str, Any]: + data = request.config.model_dump(exclude_none=True) + extensions = data.pop("extensions", {}) or {} + return {**extensions, **data} + + +def _validate_one_source(part: Any) -> None: + sources = {name: getattr(part, name) for name in ("data", "url", "file_id", "path") if getattr(part, name) is not None} + class_name = type(part).__name__ + if len(sources) != 1: + raise ValueError(f"{class_name} requires exactly one of data, url, file_id, or path.") + name, value = next(iter(sources.items())) + if isinstance(value, str) and not value: + raise ValueError(f"{class_name}.{name} must be non-empty.") + + +def _coerce_message(value: dict[str, Any] | LMMessage) -> LMMessage: + if isinstance(value, LMMessage): + return value + return LMMessage(**value) + + +def _messages_from_items(items: tuple[Any, ...], *, prompt: str | None = None) -> tuple[list[LMMessage], list[Any]]: + # TODO: Normalize DSPy-specific LM(...) objects in the LM call layer before building LMRequest. + if prompt is not None: + items = (prompt, *items) + if not items: + items = ("",) + + if len(items) == 1 and _is_message_sequence(items[0]): + items = tuple(items[0]) + + if all(isinstance(item, LMMessage) or isinstance(item, LMResponse) for item in items): + messages: list[LMMessage] = [] + for item in items: + if isinstance(item, LMMessage): + messages.append(item) + else: + messages.extend(_messages_from_response(item)) + return messages, [] + + parts = [_coerce_part(item) for item in items] + return [LMMessage(role="user", parts=parts)], [] + + +def _messages_from_response(response: LMResponse) -> list[LMMessage]: + return [LMMessage(role="assistant", parts=output.parts) for output in response.outputs] + + +def _is_message_sequence(value: Any) -> bool: + return isinstance(value, (list, tuple)) and all( + isinstance(item, LMMessage) or isinstance(item, LMResponse) for item in value + ) + + +def _coerce_part(value: Any) -> LMPart: + if isinstance( + value, + ( + LMTextPart, + LMImagePart, + LMAudioPart, + LMVideoPart, + LMDocumentPart, + LMBinaryPart, + LMToolCallPart, + LMToolResultPart, + LMThinkingPart, + LMCitationPart, + LMRefusalPart, + ), + ): + return value + if isinstance(value, str): + return LMTextPart(text=value) + if isinstance(value, dict) and "type" in value: + return pydantic.TypeAdapter(LMPart).validate_python(value) + raise TypeError(f"Cannot convert {type(value)!r} to an LMPart.") + + +def _parts_from_openai_content(content: Any) -> list[LMPart]: + if content is None: + return [] + if isinstance(content, str): + return [LMTextPart(text=content)] + if not isinstance(content, list): + return [_coerce_part(content)] + + parts = [] + for item in content: + item_type = item.get("type") if isinstance(item, dict) else None + if item_type == "text": + parts.append(LMTextPart(text=item.get("text", ""))) + elif item_type == "image_url": + image = item.get("image_url", {}) + if not isinstance(image, dict): + raise TypeError("Image content block must be a mapping.") + url = image.get("url") + if url is None: + raise ValueError("Image content block requires url.") + parts.append(_image_source_to_part(url)) + elif item_type == "input_audio": + audio = item.get("input_audio", {}) + parts.append(_audio_dict_to_part(audio)) + elif item_type == "file": + parts.append(_binary_dict_to_part(item.get("file", {}))) + elif item_type == "document": + parts.append(_document_dict_to_part(item)) + elif item_type == "video": + video = item.get("video", {}) + parts.append(_media_dict_to_video_part(video)) + else: + parts.append(_coerce_part(item)) + return parts + + +def _tool_calls_from_openai(tool_calls: list[Any]) -> list[LMToolCallPart]: + return [_tool_call_from_openai(tool_call) for tool_call in tool_calls] + + +def _tool_call_from_openai(tool_call: Any) -> LMToolCallPart: + if not isinstance(tool_call, Mapping): + part = _coerce_part(tool_call) + if isinstance(part, LMToolCallPart): + return part + raise TypeError(f"Cannot convert {type(tool_call)!r} to an LMToolCallPart.") + + function = tool_call.get("function", {}) + if not isinstance(function, Mapping): + function = {} + + args = function.get("arguments", {}) + if isinstance(args, str): + args = _parse_json_object(args) + elif isinstance(args, Mapping): + args = dict(args) + else: + args = {} + + return LMToolCallPart( + id=tool_call.get("id"), + name=function.get("name") or tool_call.get("name") or "", + args=args, + ) + + +def _image_source_to_part(source: str) -> LMImagePart: + if not isinstance(source, str): + raise TypeError("Image URL must be a string.") + if source.startswith("data:"): + media_type, data = _split_data_uri(source) + return LMImagePart(data=data, media_type=media_type) + media_type = mimetypes.guess_type(urlparse(source).path)[0] or "image/png" + return LMImagePart(url=source, media_type=media_type) + + +def _audio_dict_to_part(audio: dict[str, Any]) -> LMAudioPart: + if not isinstance(audio, dict): + raise TypeError("Audio content block must be a mapping.") + + audio_format = audio.get("format") or "wav" + if not isinstance(audio_format, str): + raise TypeError("Audio format must be a string.") + media_type = audio_format if "/" in audio_format else f"audio/{audio_format}" + + if audio.get("data") is not None: + data = audio["data"] + if isinstance(data, str) and data.startswith("data:"): + media_type, data = _split_data_uri(data) + return LMAudioPart(data=data, media_type=media_type) + if audio.get("url") is not None: + return LMAudioPart(url=audio["url"], media_type=media_type) + if audio.get("file_id") is not None: + return LMAudioPart(file_id=audio["file_id"], media_type=media_type) + if audio.get("path") is not None: + return LMAudioPart(path=audio["path"], media_type=media_type) + raise ValueError("Audio content block requires data, url, file_id, or path.") + + +def _binary_dict_to_part(file: dict[str, Any]) -> LMBinaryPart: + if file.get("file_data") is not None: + media_type, data = _split_data_uri(file["file_data"]) + return LMBinaryPart(data=data, media_type=media_type, filename=file.get("filename")) + if file.get("data") is not None: + media_type, data = _split_data_uri(file["data"]) + return LMBinaryPart(data=data, media_type=media_type, filename=file.get("filename")) + if file.get("file_id") is not None: + return LMBinaryPart(file_id=file["file_id"], filename=file.get("filename")) + raise ValueError("Binary content block requires data, file_data, or file_id.") + + +def _document_dict_to_part(item: dict[str, Any]) -> LMDocumentPart: + common = {"title": item.get("title"), "context": item.get("context")} + media_type = item.get("media_type") or "application/pdf" + for source_key in ("data", "url", "file_id", "path"): + if item.get(source_key) is not None: + return LMDocumentPart(**{source_key: item[source_key]}, media_type=media_type, **common) + + source = item.get("source") + if isinstance(source, dict): + return LMDocumentPart( + source=source, + citations=item.get("citations") or {}, + **common, + ) + if isinstance(source, str): + kwargs = _media_source_kwargs(source, default_media_type=media_type) + return LMDocumentPart(**kwargs, **common) + raise ValueError("Document content block requires source.") + + +def _media_dict_to_video_part(video: dict[str, Any]) -> LMVideoPart: + if video.get("data") is not None: + data = video["data"] + if isinstance(data, str) and data.startswith("data:"): + media_type, data = _split_data_uri(data) + else: + media_type = video.get("media_type") or "video/mp4" + return LMVideoPart(data=data, media_type=media_type) + if video.get("url") is not None: + return LMVideoPart(url=video["url"], media_type=video.get("media_type") or "video/mp4") + if video.get("file_id") is not None: + return LMVideoPart(file_id=video["file_id"], media_type=video.get("media_type") or "video/mp4") + if video.get("path") is not None: + return LMVideoPart(path=video["path"], media_type=video.get("media_type") or "video/mp4") + raise ValueError("Video content block requires data, url, file_id, or path.") + + +def _split_data_uri(value: str) -> tuple[str, str]: + if not value.startswith("data:") or "," not in value: + return "application/octet-stream", value + header, data = value.split(",", 1) + media_type = header.removeprefix("data:").split(";", 1)[0] + return media_type, data + + +def _media_source_kwargs(source: str, *, default_media_type: str) -> dict[str, str]: + if source.startswith("data:"): + media_type, data = _split_data_uri(source) + return {"data": data, "media_type": media_type} + + parsed = urlparse(source) + if parsed.scheme in {"http", "https"}: + media_type = mimetypes.guess_type(parsed.path)[0] or default_media_type + return {"url": source, "media_type": media_type} + + return {"file_id": source, "media_type": default_media_type} + + +def _coerce_tool_spec(tool: Any) -> LMToolSpec: + if isinstance(tool, LMToolSpec): + return tool + if hasattr(tool, "to_lm_tool_spec"): + return tool.to_lm_tool_spec() + if isinstance(tool, dict): + if "function" in tool: + function = tool["function"] + return LMToolSpec( + name=function.get("name"), + description=function.get("description"), + parameters=function.get("parameters", {}), + ) + return LMToolSpec(**tool) + raise TypeError(f"Cannot convert {type(tool)!r} to LMToolSpec.") + + +def _requires_output_dict(output: LMOutput) -> bool: + return bool( + output.logprobs is not None or output.reasoning_content is not None or output.tool_calls or output.citations + ) + + +def _part_to_value(part: LMPart) -> Any: + if isinstance(part, LMTextPart): + return part.text + if isinstance(part, LMThinkingPart): + return part + if isinstance(part, LMToolCallPart): + return part + if isinstance(part, LMRefusalPart): + return part.text + return part + + +def _finalize_stream_part(part: LMPart) -> LMPart: + if isinstance(part, LMToolCallPart) and "args_buffer" in part.provider_data: + return part.model_copy(update={"args": _parse_json_object_strict(part.provider_data["args_buffer"])}) + return part + + +def _tool_call_to_provider_dict(call: LMToolCallPart) -> dict[str, Any]: + data = { + "type": "function", + "function": { + "name": call.name, + "arguments": json.dumps(call.args), + }, + } + if call.id is not None: + data["id"] = call.id + return data + + +def _parse_json_object(value: str) -> dict[str, Any]: + if not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _parse_json_object_strict(value: str) -> dict[str, Any]: + if not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError("Streamed tool-call arguments must be a JSON object.") from exc + if not isinstance(parsed, dict): + raise ValueError("Streamed tool-call arguments must be a JSON object.") + return parsed diff --git a/tests/core/test_types.py b/tests/core/test_types.py new file mode 100644 index 0000000000..73a5ca3661 --- /dev/null +++ b/tests/core/test_types.py @@ -0,0 +1,316 @@ +import pydantic +import pytest + +from dspy.core.types import ( + LMAudioPart, + LMConfig, + LMDocumentPart, + LMHistoryEntry, + LMMessage, + LMOutput, + LMOutputBuilder, + LMPromptCacheConfig, + LMReasoningConfig, + LMRequest, + LMRequestPatch, + LMResponse, + LMStreamDeltaEvent, + LMTextDelta, + LMThinkingDelta, + LMThinkingPart, + LMToolCallDelta, + LMToolCallPart, + LMToolChoice, + LMToolResultPart, + LMUsage, + LMVideoPart, + User, +) + + +def _history_entry(message: LMMessage) -> LMHistoryEntry: + return LMHistoryEntry( + request=LMRequest(model="model", messages=[message]), + response=LMResponse.from_text("ok"), + timestamp="timestamp", + uuid="uuid", + ) + + +def test_message_content_and_tool_calls_normalize_for_dspy_history_surface(): + message = LMMessage( + role="assistant", + content="Use search.", + tool_calls=[ + { + "id": "call_1", + "function": {"name": "search", "arguments": '{"query": "dspy"}'}, + } + ], + ) + + assert message.text == "Use search." + assert [part for part in message.parts if isinstance(part, LMToolCallPart)] == [ + LMToolCallPart(id="call_1", name="search", args={"query": "dspy"}) + ] + assert _history_entry(message).messages == [ + { + "role": "assistant", + "content": "Use search.", + "tool_calls": [ + { + "type": "function", + "function": {"name": "search", "arguments": '{"query": "dspy"}'}, + "id": "call_1", + } + ], + } + ] + + +def test_tool_result_content_none_normalizes_to_empty_parts(): + assert LMToolResultPart(content=None).content == [] + + message = LMMessage(role="tool", content=None, tool_call_id="call_1", name="search") + result = message.parts[0] + + assert isinstance(result, LMToolResultPart) + assert result.call_id == "call_1" + assert result.name == "search" + assert result.content == [] + assert _history_entry(message).messages == [ + {"role": "tool", "content": "", "tool_call_id": "call_1", "name": "search"} + ] + + +def test_audio_content_accepts_url_and_history_preserves_url(): + message = LMMessage( + role="user", + content=[ + { + "type": "input_audio", + "input_audio": {"url": "https://example.com/audio.wav", "format": "wav"}, + } + ], + ) + + audio = message.parts[0] + assert isinstance(audio, LMAudioPart) + assert audio.url == "https://example.com/audio.wav" + assert _history_entry(message).messages[0]["content"][0]["input_audio"] == { + "format": "wav", + "url": "https://example.com/audio.wav", + } + + +def test_audio_content_defaults_null_format_to_wav(): + message = LMMessage( + role="user", + content=[ + { + "type": "input_audio", + "input_audio": {"url": "https://example.com/audio.wav", "format": None}, + } + ], + ) + + audio = message.parts[0] + + assert isinstance(audio, LMAudioPart) + assert audio.media_type == "audio/wav" + + +def test_image_content_requires_mapping_with_url(): + with pytest.raises(TypeError, match="Image content block"): + LMMessage(role="user", content=[{"type": "image_url", "image_url": "https://example.com/image.png"}]) + + with pytest.raises(ValueError, match="requires url"): + LMMessage(role="user", content=[{"type": "image_url", "image_url": {}}]) + + +def test_video_data_round_trips_through_history_messages(): + message = User(LMVideoPart(data="YWJj", media_type="video/mp4")) + content = _history_entry(message).messages[0]["content"][0] + round_tripped = LMMessage(role="user", content=[content]).parts[0] + + assert content == { + "type": "video", + "video": {"media_type": "video/mp4", "data": "data:video/mp4;base64,YWJj"}, + } + assert isinstance(round_tripped, LMVideoPart) + assert round_tripped.data == "YWJj" + assert round_tripped.media_type == "video/mp4" + + +def test_document_source_url_stays_url_and_round_trips_through_history_messages(): + message = LMMessage( + role="user", + content=[ + { + "type": "document", + "source": "https://example.com/report.pdf", + "title": "Report", + } + ], + ) + document = message.parts[0] + content = _history_entry(message).messages[0]["content"][0] + round_tripped = LMMessage(role="user", content=[content]).parts[0] + + assert isinstance(document, LMDocumentPart) + assert document.url == "https://example.com/report.pdf" + assert document.data is None + assert isinstance(round_tripped, LMDocumentPart) + assert round_tripped.url == "https://example.com/report.pdf" + assert round_tripped.data is None + + +def test_config_extensions_flatten_when_converted_to_legacy_kwargs(): + config = LMConfig.from_kwargs(temperature=0.2, provider_flag=True) + patch = LMRequestPatch(config=config) + request = LMRequest.from_call(model="model", prompt="hi", temperature=0.2, provider_flag=True) + entry = LMHistoryEntry( + request=request, + response=LMResponse.from_text("ok"), + timestamp="timestamp", + uuid="uuid", + ) + + assert patch.as_lm_kwargs() == {"provider_flag": True, "temperature": 0.2} + assert entry.kwargs == {"provider_flag": True, "temperature": 0.2} + + +def test_lm_kwargs_aliases_normalize_for_existing_dspy_lm_callers(): + config = LMConfig.from_kwargs( + reasoning={"effort": "low", "summary": "auto"}, + reasoning_effort="high", + tool_choice={"mode": "auto"}, + parallel_tool_calls=False, + cache=True, + rollout_id=7, + prompt_cache=True, + prompt_cache_key="prompt-cache", + provider_flag=True, + ) + + assert config.reasoning.effort == "high" + assert config.reasoning.summary == "auto" + assert config.tool_choice.mode == "auto" + assert config.tool_choice.parallel is False + assert config.cache.enabled is True + assert config.cache.rollout_id == 7 + assert config.prompt_cache.enabled is True + assert config.prompt_cache.key == "prompt-cache" + assert config.extensions == {"provider_flag": True} + + +def test_nested_config_aliases_remain_supported_for_existing_interfaces(): + reasoning = LMReasoningConfig.from_value({"reasoning_effort": "high"}) + tool_choice = LMToolChoice.from_value({"parallel_tool_calls": False}) + prompt_cache = LMPromptCacheConfig.from_value({"prompt_cache_key": "prompt-cache"}) + + assert reasoning.effort == "high" + assert tool_choice.mode == "auto" + assert tool_choice.parallel is False + assert prompt_cache.key == "prompt-cache" + + +def test_usage_normalizes_existing_user_visible_token_aliases(): + provider_usage = LMUsage(prompt_tokens=1, completion_tokens=2) + canonical_usage = LMUsage(input_tokens=1, output_tokens=2) + + assert provider_usage.input_tokens == 1 + assert provider_usage.output_tokens == 2 + assert provider_usage.total_tokens == 3 + assert canonical_usage.prompt_tokens == 1 + assert canonical_usage.completion_tokens == 2 + assert canonical_usage.total_tokens == 3 + + +def test_default_config_does_not_serialize_empty_stop_sequences(): + request = LMRequest.from_call(model="model", prompt="hi") + entry = LMHistoryEntry( + request=request, + response=LMResponse.from_text("ok"), + timestamp="timestamp", + uuid="uuid", + ) + + assert request.config.stop is None + assert entry.kwargs == {} + + +def test_history_entry_exposes_existing_dict_style_properties(): + message = User("hi") + request = LMRequest.from_call(model="model", messages=[message], temperature=0.2) + response = LMResponse.from_text("ok", model="response-model", usage={"input_tokens": 1}, cost=0.5) + entry = LMHistoryEntry(request=request, response=response, timestamp="timestamp", uuid="uuid") + + assert entry.model == "model" + assert entry.prompt == "hi" + assert entry.messages == [{"role": "user", "content": "hi"}] + assert entry.outputs == ["ok"] + assert entry.usage["input_tokens"] == 1 + assert entry.cost == 0.5 + assert entry.kwargs == {"temperature": 0.2} + assert entry.response_model == "response-model" + assert entry["messages"] == [{"role": "user", "content": "hi"}] + assert entry["kwargs"] == {"temperature": 0.2} + + +def test_response_rejects_empty_outputs(): + with pytest.raises(pydantic.ValidationError): + LMResponse(model="model", outputs=[]) + + +def test_output_to_value_preserves_redacted_thinking_part(): + thinking = LMThinkingPart(text="hidden", redacted=True) + output = LMOutput(parts=[thinking]) + + assert output.to_value() == [thinking] + + +def test_stream_event_indices_must_be_non_negative(): + with pytest.raises(pydantic.ValidationError): + LMStreamDeltaEvent(output_index=-1, part_index=0, delta=LMTextDelta(text="x")) + + with pytest.raises(pydantic.ValidationError): + LMStreamDeltaEvent(output_index=0, part_index=-1, delta=LMTextDelta(text="x")) + + +def test_stream_builder_rejects_sparse_output_indices(): + builder = LMOutputBuilder() + builder.apply(LMStreamDeltaEvent(output_index=2, part_index=0, delta=LMTextDelta(text="third"))) + + with pytest.raises(ValueError, match="output indices"): + builder.to_response() + + +def test_stream_builder_rejects_sparse_part_indices(): + builder = LMOutputBuilder() + builder.apply(LMStreamDeltaEvent(output_index=0, part_index=1, delta=LMTextDelta(text="second"))) + + with pytest.raises(ValueError, match="part indices"): + builder.to_response() + + +def test_stream_builder_rejects_delta_type_changes(): + builder = LMOutputBuilder() + builder.apply(LMStreamDeltaEvent(output_index=0, part_index=0, delta=LMTextDelta(text="text"))) + + with pytest.raises(ValueError, match="thinking delta"): + builder.apply(LMStreamDeltaEvent(output_index=0, part_index=0, delta=LMThinkingDelta(text="thought"))) + + +def test_stream_builder_rejects_incomplete_tool_call_arguments(): + builder = LMOutputBuilder() + builder.apply( + LMStreamDeltaEvent( + output_index=0, + part_index=0, + delta=LMToolCallDelta(id="call_1", name="search", args_delta='{"query": '), + ) + ) + + with pytest.raises(ValueError, match="tool-call arguments"): + builder.to_response() From b16d109567cfb9d1b8efe341da59e3ff67f86e14 Mon Sep 17 00:00:00 2001 From: Zhaocun Sun Date: Fri, 22 May 2026 20:54:50 +0800 Subject: [PATCH 07/23] docs: fix FAQ learn guide link (#9798) --- docs/docs/faqs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/faqs.md b/docs/docs/faqs.md index 3a5616fee9..65615faba1 100644 --- a/docs/docs/faqs.md +++ b/docs/docs/faqs.md @@ -19,7 +19,7 @@ The **DSPy** philosophy and abstraction differ significantly from other librarie ## Basic Usage -**How should I use DSPy for my task?** We wrote a [eight-step guide](learn/index.md) on this. In short, using DSPy is an iterative process. You first define your task and the metrics you want to maximize, and prepare a few example inputs — typically without labels (or only with labels for the final outputs, if your metric requires them). Then, you build your pipeline by selecting built-in layers (`modules`) to use, giving each layer a `signature` (input/output spec), and then calling your modules freely in your Python code. Lastly, you use a DSPy `optimizer` to compile your code into high-quality instructions, automatic few-shot examples, or updated LM weights for your LM. +**How should I use DSPy for my task?** We wrote an [eight-step guide](/learn/) on this. In short, using DSPy is an iterative process. You first define your task and the metrics you want to maximize, and prepare a few example inputs — typically without labels (or only with labels for the final outputs, if your metric requires them). Then, you build your pipeline by selecting built-in layers (`modules`) to use, giving each layer a `signature` (input/output spec), and then calling your modules freely in your Python code. Lastly, you use a DSPy `optimizer` to compile your code into high-quality instructions, automatic few-shot examples, or updated LM weights for your LM. **How do I convert my complex prompt into a DSPy pipeline?** See the same answer above. From efb7fee7463d0a22ae72a84be5781204c6e6dea0 Mon Sep 17 00:00:00 2001 From: Isaac Miller <17116851+isaacbmiller@users.noreply.github.com> Date: Fri, 22 May 2026 12:46:46 -0400 Subject: [PATCH 08/23] Isolate DSPy cache in tests (#9795) --- tests/conftest.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 81680d970e..aabbe43faa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,8 @@ import copy import os +from collections.abc import Iterator +from pathlib import Path +from typing import Any import pytest @@ -8,16 +11,31 @@ SKIP_DEFAULT_FLAGS = ["reliability", "extra", "llm_call", "deno"] -@pytest.fixture(autouse=True) -def clear_settings(): - """Ensures that the settings are cleared after each test.""" +def _close_cache(cache: Any) -> None: + disk_cache = getattr(cache, "disk_cache", None) + if hasattr(disk_cache, "close"): + disk_cache.close() - yield +@pytest.fixture(autouse=True) +def clear_settings(tmp_path: Path) -> Iterator[None]: + """Ensure each test gets fresh DSPy settings and an isolated cache.""" import dspy - from dspy.dsp.utils.settings import DEFAULT_CONFIG - dspy.configure(**copy.deepcopy(DEFAULT_CONFIG), inherit_config=False) + original_cache = dspy.cache + dspy.configure_cache(disk_cache_dir=tmp_path / ".dspy_cache") + try: + yield + finally: + from dspy.dsp.utils.settings import DEFAULT_CONFIG + + try: + dspy.configure(**copy.deepcopy(DEFAULT_CONFIG), inherit_config=False) + finally: + try: + _close_cache(dspy.cache) + finally: + dspy.cache = original_cache @pytest.fixture From ee2c5803a20eaf4fb0dad751c6f40da3b6099197 Mon Sep 17 00:00:00 2001 From: Maxime Rivest Date: Sat, 23 May 2026 07:02:33 -0400 Subject: [PATCH 09/23] Introduce normalized LM boundary for adapters (#9802) Moving adapters toward normalized LM requests and responses. Adapters still format prompts the same way, and current `BaseLM` calls still receive OpenAI/LiteLLM-shaped kwargs. The difference is that the adapter call path now passes through a normalized boundary internally: ```text adapter messages -> LMRequest -> OpenAI/LiteLLM kwargs -> current BaseLM -> LMResponse -> existing adapter postprocess path ``` ## Why Today adapters talk directly in provider-shaped messages and legacy output objects. That makes it hard to move tools, reasoning, citations, multimodal inputs, and future adapter types onto one LM type system. This PR adds the seam without changing parser behavior yet. We normalize before and after the LM call, then convert back to the existing postprocess shape so this can stay behavior-preserving. ## Changes - Add `dspy.clients.openai_format` for normalized `LMRequest` / `LMResponse` conversion to and from OpenAI/LiteLLM-shaped data. - Add `dspy.adapters._legacy_type_markers` for the old custom type marker expansion. - Update `Adapter.__call__()` and `acall()` to route through `LMRequest` and `LMResponse`. - Preserve unknown legacy content blocks during round-trip conversion. - Preserve text block metadata when coercing OpenAI-style content into `LMTextPart`. - Add TODOs in `base.py` marking the follow-up adapter plan and direct `LMResponse` parsing work. ## Notes `dspy.clients.openai_format` is used instead of an adapter-local bridge because provider-format conversion belongs in the client/LM layer. Later LM backends can reuse the same formatter. The conversion back from `LMResponse` to legacy outputs is temporary. A follow-up PR should replace `_call_preprocess()` / `_call_postprocess()` with an explicit adapter plan and direct `LMResponse` parsing. --- dspy/adapters/_legacy_type_markers.py | 145 ++++ dspy/adapters/base.py | 158 +++- dspy/clients/openai_format.py | 923 ++++++++++++++++++++++++ dspy/core/types.py | 12 +- tests/adapters/test_two_step_adapter.py | 2 + 5 files changed, 1229 insertions(+), 11 deletions(-) create mode 100644 dspy/adapters/_legacy_type_markers.py create mode 100644 dspy/clients/openai_format.py diff --git a/dspy/adapters/_legacy_type_markers.py b/dspy/adapters/_legacy_type_markers.py new file mode 100644 index 0000000000..3a3a788ac3 --- /dev/null +++ b/dspy/adapters/_legacy_type_markers.py @@ -0,0 +1,145 @@ +"""Private compatibility helpers for legacy `dspy.Type` marker serialization.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +import json_repair + +from dspy.adapters.types.base_type import CUSTOM_TYPE_END_IDENTIFIER, CUSTOM_TYPE_START_IDENTIFIER +from dspy.core.types import LMAudioPart, LMBinaryPart, LMDocumentPart, LMImagePart, LMMessage, LMPart, LMTextPart + + +def _expand_legacy_custom_type_markers_in_chat_message(message: dict[str, Any]) -> dict[str, Any]: + """Expand legacy marker payloads in an OpenAI-chat-shaped user message.""" + if message.get("role") != "user" or not isinstance(message.get("content"), str): + return message + content = message["content"] + if CUSTOM_TYPE_START_IDENTIFIER not in content: + return message + return {**message, "content": _split_legacy_custom_type_text_to_blocks(content)} + + +def _expand_legacy_custom_type_markers_in_lm_message(message: LMMessage) -> LMMessage: + """Expand legacy marker payloads in user `LMMessage`s into normalized parts.""" + if message.role != "user": + return message + expanded_parts: list[LMPart] = [] + changed = False + for part in message.parts: + if not isinstance(part, LMTextPart) or CUSTOM_TYPE_START_IDENTIFIER not in part.text: + expanded_parts.append(part) + continue + changed = True + expanded_parts.extend(_split_legacy_custom_type_text_to_parts(part.text)) + return message.model_copy(update={"parts": expanded_parts}) if changed else message + + +def _split_legacy_custom_type_text_to_blocks(text: str) -> list[dict[str, Any]]: + blocks: list[dict[str, Any]] = [] + last_end = 0 + for match in re.finditer(_marker_pattern(), text, re.DOTALL): + start, end = match.span() + if start > last_end: + blocks.append({"type": "text", "text": text[last_end:start]}) + blocks.extend(_legacy_custom_type_payload_to_blocks(match.group(1).strip())) + last_end = end + if last_end < len(text): + blocks.append({"type": "text", "text": text[last_end:]}) + return blocks + + +def _split_legacy_custom_type_text_to_parts(text: str) -> list[LMPart]: + parts: list[LMPart] = [] + last_end = 0 + for match in re.finditer(_marker_pattern(), text, re.DOTALL): + start, end = match.span() + if start > last_end: + parts.append(LMTextPart(text=text[last_end:start])) + parts.extend(_legacy_custom_type_payload_to_parts(match.group(1).strip())) + last_end = end + if last_end < len(text): + parts.append(LMTextPart(text=text[last_end:])) + return parts + + +def _legacy_custom_type_payload_to_blocks(payload: str) -> list[dict[str, Any]]: + parsed = _parse_legacy_payload(payload) + if isinstance(parsed, list): + return [block if isinstance(block, dict) else {"type": "text", "text": str(block)} for block in parsed] + return [{"type": "text", "text": payload}] + + +def _legacy_custom_type_payload_to_parts(payload: str) -> list[LMPart]: + parsed = _parse_legacy_payload(payload) + if not isinstance(parsed, list): + return [LMTextPart(text=payload)] + return [_legacy_content_block_to_lm_part(block) for block in parsed] + + +def _parse_legacy_payload(payload: str) -> Any: + for parse_fn in (json.loads, _parse_doubly_quoted_json, json_repair.loads): + try: + return parse_fn(payload) + except Exception: + continue + return None + + +def _parse_doubly_quoted_json(value: str) -> Any: + # Legacy `Type` payloads can be JSON-encoded twice when the serialized + # marker string is nested inside a larger JSON value, e.g. list[Image]. + return json.loads(json.loads(f'"{value}"')) + + +def _legacy_content_block_to_lm_part(block: Any) -> LMPart: + if not isinstance(block, dict): + return LMTextPart(text=str(block)) + + block_type = block.get("type") + if block_type == "text": + return LMTextPart(text=block.get("text", "")) + if block_type == "image_url": + image_url = block.get("image_url", {}) + source = image_url.get("url") if isinstance(image_url, dict) else image_url + if isinstance(source, str) and source.startswith("data:") and "," in source: + media_type, data = _split_data_uri(source) + return LMImagePart(data=data, media_type=media_type) + return LMImagePart(url=source or "") + if block_type == "input_audio": + audio = block.get("input_audio", {}) + return LMAudioPart(data=audio.get("data", ""), media_type=f"audio/{audio.get('format', 'wav')}") + if block_type == "file": + file = block.get("file", {}) + if file.get("file_data") is not None: + media_type, data = _split_data_uri(file["file_data"]) + return LMBinaryPart(data=data, media_type=media_type, filename=file.get("filename"), metadata={"legacy_content_block": block}) + return LMBinaryPart(file_id=file.get("file_id", ""), filename=file.get("filename"), metadata={"legacy_content_block": block}) + if block_type == "document": + source = block.get("source", {}) + return LMDocumentPart( + source=source if isinstance(source, dict) else {"type": "text", "data": str(source)}, + citations=block.get("citations") or {"enabled": True}, + title=block.get("title"), + context=block.get("context"), + ) + + # Keep unknown provider-shaped blocks losslessly for the current BaseLM + # compatibility path. `openai_format.part_to_openai_blocks()` rehydrates + # this metadata before the legacy provider-shaped call. A later normalized + # type pass should replace this with an explicit opaque part or remove it + # when marker-based custom type rendering is retired. + return LMTextPart(text="", metadata={"legacy_content_block": block}) + + +def _marker_pattern() -> str: + return rf"{CUSTOM_TYPE_START_IDENTIFIER}(.*?){CUSTOM_TYPE_END_IDENTIFIER}" + + +def _split_data_uri(value: str) -> tuple[str, str]: + if value.startswith("data:") and "," in value: + header, data = value.split(",", 1) + return header.removeprefix("data:").split(";", 1)[0], data + return "application/octet-stream", value diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index 7520856182..efab5c29b3 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -1,13 +1,23 @@ +import json import logging from typing import Any, get_origin import json_repair +from dspy.adapters._legacy_type_markers import ( + _expand_legacy_custom_type_markers_in_chat_message, + _expand_legacy_custom_type_markers_in_lm_message, +) from dspy.adapters.types import History, Type -from dspy.adapters.types.base_type import split_message_content_for_custom_types from dspy.adapters.types.reasoning import Reasoning from dspy.adapters.types.tool import Tool, ToolCalls from dspy.clients.base_lm import BaseLM +from dspy.clients.openai_format import ( + legacy_outputs_from_lm_response, + lm_response_from_legacy_outputs, + to_openai_chat_request, +) +from dspy.core.types import LMMessage, LMRequest, LMResponse from dspy.experimental import Citations from dspy.signatures.signature import Signature from dspy.utils.callback import BaseCallback, with_callbacks @@ -70,6 +80,12 @@ def _call_preprocess( signature: type[Signature], inputs: dict[str, Any], ) -> type[Signature]: + # TODO(adapters-plan): This remains the pre-normalized planning hook. It + # mutates `lm_kwargs` and returns only the render signature, which loses + # information we will need for plan-driven rendering/parsing. The next + # stacked PR should replace this with an explicit `_AdapterPlan` that + # records deleted fields, native tools, native output fields, inserted + # messages/parts, and LM config patches. if self.use_native_function_calling: tool_call_input_field_name = self._get_tool_call_input_field_name(signature) tool_call_output_field_name = self._get_tool_call_output_field_name(signature) @@ -96,6 +112,9 @@ def _call_preprocess( return signature_for_native_function_calling + # TODO(adapters-plan): Built-in/native response planning should move out + # of `Type.adapt_to_native_lm_feature()` and into adapter-owned planning + # renderers. Keep this compatibility hook for this boundary-only PR. # Handle custom types that use native LM features, e.g., reasoning, citations, etc. for name, field in signature.output_fields.items(): if ( @@ -115,6 +134,11 @@ def _call_postprocess( lm: BaseLM, lm_kwargs: dict[str, Any], ) -> list[dict[str, Any]]: + # TODO(adapters-plan): This still parses legacy adapter output objects. + # PR1 normalizes the LM boundary, then immediately converts back to this + # shape to avoid changing parser semantics. A later PR should parse + # `LMResponse` directly and merge text-parsed fields with explicit + # native fields from `_AdapterPlan`. values = [] tool_call_output_field_name = self._get_tool_call_output_field_name(original_signature) @@ -157,6 +181,9 @@ def _call_postprocess( ] value[tool_call_output_field_name] = ToolCalls.from_dict_list(tool_calls) + # TODO(adapter-types): Once `Type.parse_lm_output(context, output)` is + # the real normalized hook, this should not call the legacy + # provider-shaped `parse_lm_response()` directly. # Parse custom types that does not rely on the `Adapter.parse()` method for name, field in original_signature.output_fields.items(): if ( @@ -175,6 +202,112 @@ def _call_postprocess( return values + def _render_request( + self, + lm: BaseLM, + lm_kwargs: dict[str, Any], + messages: list[LMMessage | dict[str, Any]], + ) -> LMRequest: + """Build the normalized LM request for the current adapter call path. + + TODO(adapters-plan): This currently receives already-rendered messages. + Once planning lands, this should render from `_AdapterPlan` and apply + planned message/part insertions before creating `LMRequest`. + """ + return LMRequest.from_call( + model=lm.model, + messages=self._coerce_lm_messages(messages), + **lm_kwargs, + ) + + def _call_lm(self, lm: BaseLM, request: LMRequest) -> LMResponse: + """Call current `BaseLM` through the normalized request/response boundary. + + TODO(language-models): When `BaseLM` is replaced by/updated to the + normalized `BaseLM.forward(request: LMRequest) -> LMResponse` contract, + remove this compatibility shim and let adapters call the normalized LM + entry point directly. The OpenAI-shaped compatibility kwargs should live + only inside concrete LM backends. + """ + data = self._legacy_call_kwargs(request) + outputs = lm(messages=data.pop("messages"), **data) + return self._normalize_legacy_outputs(outputs, request) + + async def _acall_lm(self, lm: BaseLM, request: LMRequest) -> LMResponse: + """Async variant of `_call_lm`. + + TODO(language-models): Same transitional boundary as `_call_lm()`; this + should eventually call a normalized async LM method directly. + """ + data = self._legacy_call_kwargs(request) + outputs = await lm.acall(messages=data.pop("messages"), **data) + return self._normalize_legacy_outputs(outputs, request) + + def _legacy_call_kwargs(self, request: LMRequest) -> dict[str, Any]: + # TODO(language-models): Current `BaseLM` expects OpenAI/LiteLLM-shaped + # chat kwargs. We intentionally use `dspy.clients.openai_format` here so + # the conversion code lives in the future LM/client layer, not in + # adapters. Remove this adapter helper once `BaseLM` accepts `LMRequest`. + data = to_openai_chat_request(request) + data.pop("model", None) + # TODO(language-models): `cache` and `rollout_id` are DSPy BaseLM + # execution controls, not provider request fields. The future + # normalized LM base should own them before provider-format conversion. + if request.config.cache is not None: + if request.config.cache.enabled is not None: + data["cache"] = request.config.cache.enabled + if request.config.cache.rollout_id is not None: + data["rollout_id"] = request.config.cache.rollout_id + return data + + def _coerce_lm_messages(self, messages: list[LMMessage | dict[str, Any]]) -> list[LMMessage]: + """Normalize subclass `format()` output before the LM boundary. + + TODO(adapters-normalized-rendering): Adapter `format()` methods still + return OpenAI-chat-shaped dictionaries. This coercion is the bridge until + adapters render `LMMessage` / `LMPart` directly. + """ + return [ + _expand_legacy_custom_type_markers_in_lm_message( + message if isinstance(message, LMMessage) else self._chat_dict_to_lm_message(message) + ) + for message in messages + ] + + def _chat_dict_to_lm_message(self, message: dict[str, Any]) -> LMMessage: + try: + return LMMessage(**message) + except Exception: + # TODO(legacy-custom-types): Unknown OpenAI content blocks are + # temporarily preserved as `legacy_content_block` metadata on an + # empty text part so `openai_format` can round-trip them back to + # current BaseLM calls. Replace this with either explicit opaque + # provider parts or remove it when marker-based custom type + # serialization is retired. + message = dict(message) + content = message.get("content") + if isinstance(content, list): + sanitized = [] + supported = {"text", "image_url", "input_audio", "file", "document", "video"} + for block in content: + if isinstance(block, dict) and block.get("type") in supported: + sanitized.append(block) + elif isinstance(block, dict): + sanitized.append({"type": "text", "text": "", "metadata": {"legacy_content_block": block}}) + else: + sanitized.append({"type": "text", "text": json.dumps(block, ensure_ascii=False)}) + message["content"] = sanitized + return LMMessage(**message) + + def _normalize_legacy_outputs(self, outputs: list[dict[str, Any] | str | None], request: LMRequest) -> LMResponse: + """Convert current `BaseLM` outputs into a normalized `LMResponse`. + + TODO(language-models): Current `BaseLM` returns `list[str | dict | None]`. + Future LMs should return `LMResponse` directly, making this method a + compatibility-only path for old/custom LMs. + """ + return lm_response_from_legacy_outputs(outputs, request) + def __call__( self, lm: BaseLM, @@ -200,9 +333,14 @@ def __call__( signature's output field names. For multiple generations (n > 1), returns multiple dictionaries. """ processed_signature = self._call_preprocess(lm, lm_kwargs, signature, inputs) - inputs = self.format(processed_signature, demos, inputs) - - outputs = lm(messages=inputs, **lm_kwargs) + messages = self.format(processed_signature, demos, inputs) + request = self._render_request(lm, lm_kwargs, messages) + response = self._call_lm(lm, request) + # TODO(adapters-response): We normalize at the LM boundary, but still + # convert back to legacy postprocess dictionaries here to keep this PR + # behavior-preserving. Replace with direct `LMResponse` parsing once the + # explicit adapter plan exists. + outputs = legacy_outputs_from_lm_response(response) return self._call_postprocess(processed_signature, signature, outputs, lm, lm_kwargs) async def acall( @@ -214,9 +352,12 @@ async def acall( inputs: dict[str, Any], ) -> list[dict[str, Any]]: processed_signature = self._call_preprocess(lm, lm_kwargs, signature, inputs) - inputs = self.format(processed_signature, demos, inputs) - - outputs = await lm.acall(messages=inputs, **lm_kwargs) + messages = self.format(processed_signature, demos, inputs) + request = self._render_request(lm, lm_kwargs, messages) + response = await self._acall_lm(lm, request) + # TODO(adapters-response): Keep in sync with `__call__()` until both use + # direct `LMResponse` parsing. + outputs = legacy_outputs_from_lm_response(response) return self._call_postprocess(processed_signature, signature, outputs, lm, lm_kwargs) def format( @@ -292,8 +433,7 @@ def format( content = self.format_user_message_content(signature, inputs_copy, main_request=True) messages.append({"role": "user", "content": content}) - messages = split_message_content_for_custom_types(messages) - return messages + return [_expand_legacy_custom_type_markers_in_chat_message(message) for message in messages] def format_system_message(self, signature: type[Signature]) -> str: """Format the system message for the LM call. diff --git a/dspy/clients/openai_format.py b/dspy/clients/openai_format.py new file mode 100644 index 0000000000..98d72d5a2d --- /dev/null +++ b/dspy/clients/openai_format.py @@ -0,0 +1,923 @@ +"""Translate between DSPy's LM types and OpenAI-shaped JSON. + +This module does not call any provider. It only maps data shapes: + +```text +LMRequest -> OpenAI Chat / Responses / text-completion kwargs +provider response -> LMResponse +``` + +The concrete OpenAI backends import these functions, then add transport, +streaming, and caching around them. Read the file in +this order when learning it: + +1. `to_openai_chat_request()` maps chat-completion requests. +2. `to_openai_responses_request()` maps Responses API requests. +3. `to_openai_text_request()` maps legacy text-completion requests. +4. `completion_to_lm_response()` and `responses_to_lm_response()` map outputs. +5. The final utility section handles media sources, data URIs, and object/dict + access. +""" + +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +from typing import Any + +import pydantic + +from dspy.core.types import ( + LMAudioPart, + LMBinaryPart, + LMCitationPart, + LMConfig, + LMDocumentPart, + LMImagePart, + LMMessage, + LMOutput, + LMRefusalPart, + LMRequest, + LMResponse, + LMTextPart, + LMThinkingPart, + LMToolCallPart, + LMToolChoice, + LMToolResultPart, + LMToolSpec, + LMUsage, + LMVideoPart, +) + +__all__ = [ + "to_openai_chat_request", + "to_openai_responses_request", + "to_openai_text_request", + "completion_to_lm_response", + "responses_to_lm_response", + "provider_tool_call_to_part", + "responses_function_call_to_part", + "usage_from_response", +] + + +# --------------------------------------------------------------------------- +# DSPy request -> OpenAI Chat Completions +# +# Chat Completions expects: +# {"model": ..., "messages": [...], "tools": [...], ...config} +# Each DSPy message stays one OpenAI message, except assistant tool calls are +# split into OpenAI's top-level `tool_calls` field on the assistant message. +# --------------------------------------------------------------------------- + + +def to_openai_chat_request(request: LMRequest) -> dict[str, Any]: + """Convert a normalized DSPy request into Chat Completions kwargs.""" + data = {"model": request.model, "messages": [message_to_openai_chat(message) for message in request.messages]} + data.update(common_config_kwargs(request.config, model=request.model, endpoint="chat")) + if request.config.tool_choice is not None: + data.update(tool_choice_to_openai(request.config.tool_choice)) + if request.tools: + data["tools"] = [tool_to_openai(tool) for tool in request.tools] + return data + + +def message_to_openai_chat(message: LMMessage) -> dict[str, Any]: + """Convert one DSPy message into one Chat Completions message.""" + output: dict[str, Any] = {"role": message.role} + if message.name is not None: + output["name"] = message.name + + if message.role == "assistant": + tool_calls = [part for part in message.parts if isinstance(part, LMToolCallPart)] + content_parts = [part for part in message.parts if not isinstance(part, LMToolCallPart)] + output["content"] = parts_to_openai_content(content_parts) + if tool_calls: + output["tool_calls"] = [assistant_tool_call_to_openai(part) for part in tool_calls] + return output + + if message.role == "tool" and len(message.parts) == 1 and isinstance(message.parts[0], LMToolResultPart): + result = message.parts[0] + output.update(tool_result_to_openai(result)) + if result.call_id is not None: + output["tool_call_id"] = result.call_id + if result.name is not None: + output["name"] = result.name + return output + + output["content"] = parts_to_openai_content(message.parts) + return output + + +# --------------------------------------------------------------------------- +# DSPy request -> OpenAI Responses +# +# Responses expects a flat `input` list, not a chat `messages` list. A single +# DSPy assistant message can therefore become two kinds of input items: one +# message item for content and one function_call item for each tool call. +# --------------------------------------------------------------------------- + + +def to_openai_responses_request(request: LMRequest) -> dict[str, Any]: + """Convert a normalized DSPy request into Responses API kwargs.""" + config = request.config + data: dict[str, Any] = { + "model": request.model, + "input": [item for message in request.messages for item in message_to_responses_input_items(message)], + } + data.update(responses_config_kwargs(config, model=request.model)) + if config.tool_choice is not None: + data.update(tool_choice_to_openai(config.tool_choice)) + if request.tools: + data["tools"] = [tool_to_openai(tool) for tool in request.tools] + return data + + +def message_to_responses_input_items(message: LMMessage) -> list[dict[str, Any]]: + """Convert one DSPy message into one or more Responses input items.""" + if message.role == "tool" and len(message.parts) == 1 and isinstance(message.parts[0], LMToolResultPart): + result = message.parts[0] + item = {"type": "function_call_output", "output": responses_tool_output_text(tool_result_to_openai(result)["content"])} + if result.call_id is not None: + item["call_id"] = result.call_id + return [item] + + tool_calls = [part for part in message.parts if isinstance(part, LMToolCallPart)] + content_parts = [part for part in message.parts if not isinstance(part, LMToolCallPart)] + content = parts_to_responses_content(content_parts) + items: list[dict[str, Any]] = [] + + if content or message.role != "assistant" or not tool_calls: + item: dict[str, Any] = {"role": message.role, "content": content} + if message.name is not None: + item["name"] = message.name + items.append(item) + + if message.role == "assistant": + items.extend(tool_call_to_responses_input(tool_call) for tool_call in tool_calls) + return items + + +def parts_to_responses_content(parts: list[Any]) -> list[dict[str, Any]]: + blocks = parts_to_openai_content(parts) + if isinstance(blocks, str): + return [{"type": "input_text", "text": blocks}] + return [content_block_to_responses(block) for block in blocks] + + +def tool_call_to_responses_input(tool_call_part: LMToolCallPart) -> dict[str, Any]: + tool_call = assistant_tool_call_to_openai(tool_call_part) + function = tool_call.get("function", {}) + item = {"type": "function_call", "name": function.get("name", ""), "arguments": function.get("arguments", "{}")} + call_id = tool_call.get("id") or tool_call.get("call_id") + if call_id is not None: + item["call_id"] = call_id + return item + + +def content_block_to_responses(block: dict[str, Any]) -> dict[str, Any]: + block_type = block.get("type") + if block_type == "text": + return {"type": "input_text", "text": block.get("text", "")} + if block_type == "image_url": + image_url = block.get("image_url", {}) + out = {"type": "input_image", "image_url": image_url.get("url", "")} + if image_url.get("detail") is not None: + out["detail"] = image_url["detail"] + return out + if block_type == "input_audio": + return block + if block_type == "file": + file = block.get("file", {}) + return { + "type": "input_file", + "file_data": file.get("file_data"), + "filename": file.get("filename"), + "file_id": file.get("file_id"), + } + return block + + +# --------------------------------------------------------------------------- +# DSPy request -> OpenAI text completions +# +# Text completions have no native message roles. We concatenate text-only +# messages with blank lines and append DSPy's historical response marker. +# --------------------------------------------------------------------------- + + +def to_openai_text_request(request: LMRequest) -> dict[str, Any]: + """Convert a normalized DSPy request into text-completion kwargs.""" + data = {"model": request.model, "prompt": messages_to_text_prompt(request.messages)} + data.update(text_config_kwargs(request.config)) + return data + + +def messages_to_text_prompt(messages: list[LMMessage]) -> str: + """Flatten text-only messages into the prompt used by text completions.""" + chunks = [] + for message in messages: + texts = [] + for part in message.parts: + if not isinstance(part, LMTextPart): + raise ValueError(f"OpenAI text completions only support text parts, but received {type(part).__name__}.") + texts.append(part.text) + chunks.append("".join(texts)) + return "\n\n".join(chunks + ["BEGIN RESPONSE:"]) + + +# --------------------------------------------------------------------------- +# Shared request mappers +# +# The three OpenAI endpoint families use slightly different outer envelopes, but most +# leaf values are the same: text parts, media parts, tools, tool choices, and +# generation config. The helpers below keep those conversions in one place. +# --------------------------------------------------------------------------- + + +def parts_to_openai_content(parts: list[Any]) -> str | list[dict[str, Any]]: + """Convert DSPy parts into OpenAI `content`. + + OpenAI accepts a bare string for the common single-text case. Mixed content + becomes a list of content blocks. + """ + if len(parts) == 1 and isinstance(parts[0], LMTextPart) and "legacy_content_block" not in parts[0].metadata: + return parts[0].text + blocks: list[dict[str, Any]] = [] + for part in parts: + blocks.extend(part_to_openai_blocks(part)) + return blocks + + +def part_to_openai_blocks(part: Any) -> list[dict[str, Any]]: + """Convert one DSPy part into one or more OpenAI content blocks.""" + legacy_block = getattr(part, "metadata", {}).get("legacy_content_block") + if legacy_block is not None: + return [dict(legacy_block)] + if isinstance(part, LMTextPart): + return [{"type": "text", "text": part.text}] + if isinstance(part, LMImagePart): + return [image_to_openai(part)] + if isinstance(part, LMDocumentPart): + return document_to_openai_blocks(part) + if isinstance(part, LMAudioPart): + return [audio_to_openai(part)] + if isinstance(part, LMVideoPart): + return [video_to_openai(part)] + if isinstance(part, LMBinaryPart): + return [binary_to_openai(part)] + if isinstance(part, LMThinkingPart): + return [{"type": "text", "text": part.text}] + if isinstance(part, LMCitationPart): + citation = " ".join(value for value in (part.title, part.text, part.url) if value) + return [{"type": "text", "text": citation}] + if isinstance(part, LMToolResultPart): + return part_to_openai_blocks(LMTextPart(text="".join(part_text(value) for value in part.content))) + return [{"type": "text", "text": str(part)}] + + +def image_to_openai(image: LMImagePart) -> dict[str, Any]: + image_url: dict[str, Any] = {"url": media_source(image)} + if image.detail is not None: + image_url["detail"] = image.detail + return {"type": "image_url", "image_url": image_url} + + +def audio_to_openai(audio: LMAudioPart) -> dict[str, Any]: + if audio.data is not None: + data = audio.data + media_type = audio.media_type + elif audio.path is not None: + data = read_path_base64(audio.path) + media_type = media_type_for_path(audio.path, fallback=audio.media_type) + else: + raise ValueError("OpenAI-format audio input requires base64 `data` or local `path`.") + return {"type": "input_audio", "input_audio": {"data": data, "format": media_format(media_type)}} + + +def document_to_openai_blocks(document: LMDocumentPart) -> list[dict[str, Any]]: + block: dict[str, Any] = {"type": "document"} + if document.source is not None: + block["source"] = document.source + else: + block["source"] = media_source(document) + block["media_type"] = document.media_type + if document.citations: + block["citations"] = document.citations + if document.title is not None: + block["title"] = document.title + if document.context is not None: + block["context"] = document.context + return [block] + + +def video_to_openai(video: LMVideoPart) -> dict[str, Any]: + filename = os.path.basename(video.path) if video.path is not None else None + return binary_to_openai( + LMBinaryPart( + data=video.data, + url=video.url, + file_id=video.file_id, + path=video.path, + media_type=video.media_type, + filename=filename, + ) + ) + + +def binary_to_openai(binary: LMBinaryPart) -> dict[str, Any]: + file_data: dict[str, Any] = {} + if binary.data is not None: + file_data["file_data"] = data_uri(binary.media_type, binary.data) + elif binary.path is not None: + file_data["file_data"] = data_uri_from_path(binary.path, fallback_media_type=binary.media_type) + file_data["filename"] = binary.filename or os.path.basename(binary.path) + elif binary.url is not None: + file_data["file_data"] = binary.url + if binary.file_id is not None: + file_data["file_id"] = binary.file_id + if binary.filename is not None: + file_data["filename"] = binary.filename + return {"type": "file", "file": file_data} + + +def tool_to_openai(tool: LMToolSpec) -> dict[str, Any]: + data = {"type": "function", "function": {"name": tool.name, "parameters": tool.parameters}} + if tool.description is not None: + data["function"]["description"] = tool.description + data.update(tool.provider_data) + return data + + +def tool_choice_to_openai(choice: LMToolChoice) -> dict[str, Any]: + if choice.allowed: + if len(choice.allowed) != 1 or choice.mode not in {"required", "auto"}: + raise ValueError( + "OpenAI-format tool_choice only supports constraining to a single allowed tool " + "with mode 'required' or 'auto'." + ) + data: dict[str, Any] = {"tool_choice": {"type": "function", "function": {"name": choice.allowed[0]}}} + else: + data = {"tool_choice": choice.mode} + if choice.parallel is not None: + data["parallel_tool_calls"] = choice.parallel + return data + + +def assistant_tool_call_to_openai(call: LMToolCallPart) -> dict[str, Any]: + data = {"type": "function", "function": {"name": call.name, "arguments": json.dumps(call.args)}} + if call.id is not None: + data["id"] = call.id + data.update(call.provider_data) + return data + + +def tool_result_to_openai(result: LMToolResultPart) -> dict[str, Any]: + return {"content": parts_to_openai_content(result.content)} + + +def common_config_kwargs(config: LMConfig, *, model: str | None = None, endpoint: str = "chat") -> dict[str, Any]: + """Convert shared DSPy config fields into Chat Completions kwargs.""" + data = dict(config.extensions) + _validate_openai_reasoning_temperature(config, model=model, endpoint=endpoint) + for key in ("temperature", "top_p"): + value = getattr(config, key) + if value is not None: + data[key] = value + if config.max_tokens is not None: + token_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" + data[token_key] = config.max_tokens + if config.stop: + data["stop"] = config.stop + if config.logprobs is not None: + data["logprobs"] = config.logprobs + if config.n is not None: + data["n"] = config.n + if config.response_format is not None: + data["response_format"] = config.response_format + if config.reasoning is not None: + data.update(reasoning_to_chat_kwargs(config.reasoning)) + if config.prompt_cache is not None: + data.update(prompt_cache_to_kwargs(config.prompt_cache)) + return data + + +def responses_config_kwargs(config: LMConfig, *, model: str | None = None) -> dict[str, Any]: + """Convert shared DSPy config fields into Responses API kwargs.""" + data = dict(config.extensions) if config.extensions else {} + _validate_openai_reasoning_temperature(config, model=model, endpoint="responses") + for key in ("temperature", "top_p"): + value = getattr(config, key) + if value is not None: + data[key] = value + if config.max_tokens is not None: + data["max_output_tokens"] = config.max_tokens + if config.n is not None: + data["n"] = config.n + if config.logprobs is not None: + data["logprobs"] = config.logprobs + if config.stop: + data["stop"] = config.stop + if config.reasoning is not None: + data.update(reasoning_to_responses_kwargs(config.reasoning)) + if config.prompt_cache is not None: + data.update(prompt_cache_to_kwargs(config.prompt_cache)) + if config.response_format is not None: + text = data.pop("text", {}) + data["text"] = {**text, "format": response_format_to_responses(config.response_format)} + return data + + +def text_config_kwargs(config: LMConfig) -> dict[str, Any]: + """Convert shared DSPy config fields into text-completion kwargs.""" + data = dict(config.extensions) + for key in ("temperature", "max_tokens", "top_p"): + value = getattr(config, key) + if value is not None: + data[key] = value + if config.stop: + data["stop"] = config.stop + if config.logprobs is not None: + data["logprobs"] = config.logprobs + if config.n is not None: + data["n"] = config.n + return data + + +def reasoning_to_chat_kwargs(reasoning: Any) -> dict[str, Any]: + data = {} + if reasoning.effort is not None: + data["reasoning_effort"] = reasoning.effort + return data + + +def reasoning_to_responses_kwargs(reasoning: Any) -> dict[str, Any]: + data = {} + if reasoning.effort is not None: + data["effort"] = reasoning.effort + if reasoning.summary is not None: + data["summary"] = reasoning.summary + return {"reasoning": data} if data else {} + + +def _validate_openai_reasoning_temperature(config: LMConfig, *, model: str | None, endpoint: str) -> None: + if not _is_openai_reasoning_model(model): + return + effort = getattr(config.reasoning, "effort", None) if config.reasoning is not None else None + if effort in {None, "none"}: + return + if config.temperature in {None, 1}: + return + + from dspy.utils.exceptions import LMUnsupportedFeatureError + + raise LMUnsupportedFeatureError( + "OpenAI reasoning models only support the default temperature when reasoning effort is active. " + "Use temperature=None or temperature=1, or set reasoning_effort='none'.", + model=model, + provider="openai", + features=["temperature", "reasoning"], + issues=[ + f"{endpoint} request used reasoning effort {effort!r} with temperature={config.temperature!r}.", + ], + ) + + +def _uses_max_completion_tokens(model: str | None) -> bool: + return _is_openai_reasoning_model(model) + + +def _is_openai_reasoning_model(model: str | None) -> bool: + if not isinstance(model, str): + return False + model_name = model.removeprefix("openai/").lower() + if "chat" in model_name: + return False + return model_name.startswith(("o1", "o3", "o4", "gpt-5")) + + +def prompt_cache_to_kwargs(cache: Any) -> dict[str, Any]: + data = {} + if cache.key is not None: + data["prompt_cache_key"] = cache.key + if cache.enabled is False: + data["prompt_cache"] = False + return data + + +def response_format_to_responses(value: Any) -> Any: + if isinstance(value, type) and issubclass(value, pydantic.BaseModel): + return {"name": value.__name__, "type": "json_schema", "schema": value.model_json_schema()} + return value + + +def responses_tool_output_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join( + block.get("text", "") if isinstance(block, dict) and block.get("type") in {"text", "input_text"} else str(block) + for block in content + ) + return str(content) + + +# --------------------------------------------------------------------------- +# OpenAI response -> DSPy response +# +# Chat and text completions return `choices`; Responses returns one `output` +# list. Both are normalized to an `LMResponse` with one or more `LMOutput`s and +# typed DSPy parts. +# --------------------------------------------------------------------------- + + +def completion_to_lm_response(response: Any, request: LMRequest) -> LMResponse: + """Convert an OpenAI Chat or text completion response into `LMResponse`.""" + choices = get_value(response, "choices", []) or [] + return LMResponse( + model=get_value(response, "model") or request.model, + outputs=[choice_to_lm_output(choice) for choice in choices], + usage=usage_from_response(response), + cache_hit=bool(get_value(response, "cache_hit", False)), + response_id=get_value(response, "id"), + provider_response=response, + ) + + +def choice_to_lm_output(choice: Any) -> LMOutput: + """Convert one completion choice into one DSPy output candidate.""" + message = get_value(choice, "message") + parts = [] + if message is not None: + reasoning = get_value(message, "reasoning_content") + if reasoning: + parts.append(LMThinkingPart(text=str(reasoning))) + content = get_value(message, "content") + if content: + parts.extend(message_content_to_parts(content)) + for tool_call in get_value(message, "tool_calls") or []: + parts.append(provider_tool_call_to_part(tool_call)) + parts.extend(extract_citations_from_choice(choice)) + else: + text = get_value(choice, "text") + if text: + parts.extend(message_content_to_parts(text)) + finish_reason = get_value(choice, "finish_reason") + return LMOutput( + parts=parts, + finish_reason=finish_reason, + truncated=finish_reason == "length", + logprobs=get_value(choice, "logprobs"), + provider_output=choice, + ) + + +def responses_to_lm_response(response: Any, request: LMRequest) -> LMResponse: + """Convert an OpenAI Responses object into `LMResponse`. + + The Responses API represents one assistant answer as a sequence of output + items: messages, function calls, reasoning, binary artifacts, images, and refusals. + DSPy stores those as typed parts on one `LMOutput`. + """ + parts = [] + for output_item in get_value(response, "output", []) or []: + output_type = get_value(output_item, "type") + if output_type == "message": + for content_item in get_value(output_item, "content", []) or []: + parts.extend(response_content_item_to_parts(content_item)) + parts.extend(responses_annotations_to_citations(content_item)) + elif output_type == "function_call": + parts.append(responses_function_call_to_part(output_item)) + elif output_type in {"image", "output_image", "image_generation_call"}: + parts.append(output_image_to_part(output_item)) + elif output_type in {"audio", "output_audio"}: + parts.append(output_audio_to_part(output_item)) + elif output_type in {"file", "output_file"}: + parts.append(output_file_to_part(output_item)) + elif output_type == "refusal": + parts.append(refusal_to_part(output_item)) + elif output_type == "reasoning": + for item in get_value(output_item, "content") or get_value(output_item, "summary") or []: + text = get_value(item, "text") + if text: + parts.append(LMThinkingPart(text=text)) + return LMResponse( + model=get_value(response, "model") or request.model, + outputs=[LMOutput(parts=parts, provider_output=response)], + usage=usage_from_response(response), + cache_hit=bool(get_value(response, "cache_hit", False)), + response_id=get_value(response, "id"), + provider_response=response, + ) + + +def message_content_to_parts(content: Any) -> list[Any]: + if isinstance(content, str): + return [LMTextPart(text=content)] + if not isinstance(content, list): + return [LMTextPart(text=str(content))] + parts = [] + for item in content: + parts.extend(response_content_item_to_parts(item)) + return parts + + +def response_content_item_to_parts(item: Any) -> list[Any]: + item_type = get_value(item, "type") + text = get_value(item, "text") + if item_type in {"text", "output_text", "input_text"} or (text is not None and item_type is None): + return [LMTextPart(text=text)] + if item_type in {"refusal", "output_refusal"}: + return [refusal_to_part(item)] + if item_type in {"image", "output_image", "image_url"}: + return [output_image_to_part(item)] + if item_type in {"audio", "output_audio", "input_audio"}: + return [output_audio_to_part(item)] + if item_type in {"file", "output_file", "input_file"}: + return [output_file_to_part(item)] + if item_type in {"tool_call", "function_call"}: + return [provider_tool_call_to_part(item)] + return [] + + +def provider_tool_call_to_part(tool_call: Any) -> LMToolCallPart: + """Convert an OpenAI-shaped tool call into a DSPy tool-call part.""" + function = get_value(tool_call, "function", {}) + name = get_value(function, "name") or get_value(tool_call, "name") + arguments = get_value(function, "arguments", get_value(tool_call, "arguments", "{}")) + provider_data = model_dump(tool_call) + try: + args = json.loads(arguments) if isinstance(arguments, str) else dict(arguments) + except Exception as error: + args = {} + provider_data["raw_arguments"] = arguments + provider_data["arguments_parse_error"] = str(error) + call_id = get_value(tool_call, "call_id") or get_value(tool_call, "id") + return LMToolCallPart(id=call_id, name=name or "", args=args, provider_data=provider_data) + + +def responses_function_call_to_part(output_item: Any) -> LMToolCallPart: + """Convert one Responses function_call item into a DSPy tool-call part.""" + args = get_value(output_item, "arguments", {}) + provider_data = model_dump(output_item) + if isinstance(args, str): + try: + args = json.loads(args) + except Exception as error: + provider_data["raw_arguments"] = args + provider_data["arguments_parse_error"] = str(error) + args = {} + return LMToolCallPart(id=get_value(output_item, "call_id"), name=get_value(output_item, "name", ""), args=args, provider_data=provider_data) + + +def citation_to_part(citation: Any) -> LMCitationPart: + if hasattr(citation, "model_dump"): + citation = citation.model_dump(exclude_none=True) + if not isinstance(citation, dict): + citation = {"text": str(citation)} + citation_fields = {"cited_text", "text", "supported_text", "document_title", "title", "url"} + return LMCitationPart( + text=citation.get("cited_text") or citation.get("text") or citation.get("supported_text"), + title=citation.get("document_title") or citation.get("title"), + url=citation.get("url"), + metadata={key: value for key, value in citation.items() if key not in citation_fields}, + ) + + +def extract_citations_from_choice(choice: Any) -> list[LMCitationPart]: + try: + message = get_value(choice, "message") + provider_specific_fields = get_value(message, "provider_specific_fields", {}) or {} + citations_data = provider_specific_fields.get("citations") + if isinstance(citations_data, list): + citations = [] + for item in citations_data: + citations.extend(item if isinstance(item, list) else [item]) + return [citation_to_part(citation) for citation in citations] + except Exception: + return [] + return [] + + +def responses_annotations_to_citations(content_item: Any) -> list[LMCitationPart]: + return [citation_to_part(annotation) for annotation in get_value(content_item, "annotations", []) or []] + + +def output_image_to_part(value: Any) -> LMImagePart: + data = model_dump(value) + image_url = data.get("image_url") + if isinstance(image_url, dict): + image_url = image_url.get("url") + source = image_url or data.get("url") + b64_data = data.get("b64_json") or data.get("data") + file_id = data.get("file_id") + media_type = data.get("media_type") or data.get("mime_type") or "image/png" + detail = data.get("detail") + if b64_data is not None: + if isinstance(b64_data, str) and b64_data.startswith("data:"): + media_type, b64_data = split_data_uri(b64_data) + return LMImagePart(data=b64_data, media_type=media_type, detail=detail) + if source is not None: + return LMImagePart(url=source, media_type=media_type, detail=detail) + if file_id is not None: + return LMImagePart(file_id=file_id, media_type=media_type, detail=detail) + raise ValueError("Provider image output did not include data, url, or file_id.") + + +def output_audio_to_part(value: Any) -> LMAudioPart: + data = model_dump(value) + audio = data.get("audio") if isinstance(data.get("audio"), dict) else data + source = audio.get("url") + b64_data = audio.get("data") or audio.get("b64_json") + file_id = audio.get("file_id") + media_type = audio.get("media_type") or audio.get("mime_type") or "audio/wav" + if b64_data is not None: + if isinstance(b64_data, str) and b64_data.startswith("data:"): + media_type, b64_data = split_data_uri(b64_data) + return LMAudioPart(data=b64_data, media_type=media_type) + if source is not None: + return LMAudioPart(url=source, media_type=media_type) + if file_id is not None: + return LMAudioPart(file_id=file_id, media_type=media_type) + raise ValueError("Provider audio output did not include data, url, or file_id.") + + +def output_file_to_part(value: Any) -> LMBinaryPart: + data = model_dump(value) + file = data.get("file") if isinstance(data.get("file"), dict) else data + source = file.get("url") + b64_data = file.get("file_data") or file.get("data") + file_id = file.get("file_id") or file.get("id") + filename = file.get("filename") + media_type = file.get("media_type") or file.get("mime_type") or "application/octet-stream" + if b64_data is not None: + if isinstance(b64_data, str) and b64_data.startswith("data:"): + media_type, b64_data = split_data_uri(b64_data) + return LMBinaryPart(data=b64_data, media_type=media_type, filename=filename) + if source is not None: + return LMBinaryPart(url=source, media_type=media_type, filename=filename) + if file_id is not None: + return LMBinaryPart(file_id=file_id, media_type=media_type, filename=filename) + raise ValueError("Provider file output did not include data, url, or file_id.") + + +def refusal_to_part(value: Any) -> LMRefusalPart: + text = get_value(value, "refusal") or get_value(value, "text") or get_value(value, "content") or str(value) + return LMRefusalPart(text=str(text)) + + +def cost_from_response(response: Any) -> float | None: + hidden = getattr(response, "_hidden_params", None) or {} + return hidden.get("response_cost") if isinstance(hidden, dict) else None + + +def usage_from_response(response: Any) -> LMUsage | None: + """Convert provider usage objects or dictionaries into `LMUsage`.""" + usage = get_value(response, "usage") + if usage is None: + return None + if hasattr(usage, "model_dump"): + usage = usage.model_dump(exclude_none=True) + elif not isinstance(usage, dict): + data = {} + for key in dir(usage): + if key.startswith("_"): + continue + try: + value = getattr(usage, key) + except Exception: + continue + if value is not None and not callable(value): + data[key] = value + usage = data + return LMUsage(**dict(usage)) + + +# --------------------------------------------------------------------------- +# Small utilities +# --------------------------------------------------------------------------- + + +def media_source(part: LMImagePart | LMAudioPart | LMDocumentPart | LMBinaryPart) -> str: + if part.data is not None: + return data_uri(part.media_type, part.data) + if part.url is not None: + return part.url + if part.file_id is not None: + return part.file_id + if part.path is not None: + return data_uri_from_path(part.path, fallback_media_type=part.media_type) + raise ValueError(f"{type(part).__name__} has no media source.") + + +def read_path_base64(path: str) -> str: + with open(path, "rb") as file: + return base64.b64encode(file.read()).decode("ascii") + + +def media_type_for_path(path: str, *, fallback: str) -> str: + return mimetypes.guess_type(path)[0] or fallback + + +def data_uri_from_path(path: str, *, fallback_media_type: str) -> str: + return data_uri(media_type_for_path(path, fallback=fallback_media_type), read_path_base64(path)) + + +def data_uri(media_type: str, data: str) -> str: + if data.startswith("data:"): + return data + return f"data:{media_type};base64,{data}" + + +def split_data_uri(value: str) -> tuple[str, str]: + if not value.startswith("data:") or "," not in value: + return "application/octet-stream", value + header, data = value.split(",", 1) + media_type = header.removeprefix("data:").split(";", 1)[0] + return media_type, data + + +def media_format(media_type: str) -> str: + format_ = media_type.split("/", 1)[1] if "/" in media_type else media_type + return {"x-wav": "wav", "mpeg": "mp3"}.get(format_, format_) + + +def part_text(value: Any) -> str: + return value.text if isinstance(value, LMTextPart) else str(value) + + +def get_value(value: Any, key: str, default: Any = None) -> Any: + return value.get(key, default) if isinstance(value, dict) else getattr(value, key, default) + + +def model_dump(value: Any) -> dict[str, Any]: + if hasattr(value, "model_dump"): + return value.model_dump(exclude_none=True) + if isinstance(value, dict): + return dict(value) + data = {} + for key in ("id", "call_id", "type", "name", "arguments", "status", "text", "refusal", "url", "data", "file_id", "filename", "media_type", "mime_type"): + item = getattr(value, key, None) + if item is not None: + data[key] = item + return data + + +# --------------------------------------------------------------------------- +# Current BaseLM adapter compatibility helpers +# +# These helpers keep today's adapter->BaseLM boundary normalized internally while +# BaseLM still returns legacy `list[str | dict | None]` outputs. Future native +# LanguageModel implementations should use completion_to_lm_response() / +# responses_to_lm_response() directly. +# --------------------------------------------------------------------------- + + +def lm_response_from_legacy_outputs(outputs: list[dict[str, Any] | str | None], request: LMRequest) -> LMResponse: + """Normalize current legacy `BaseLM` outputs into an `LMResponse`.""" + if not outputs: + return LMResponse(model=request.model, outputs=[LMOutput(parts=[], metadata={"empty_legacy_outputs": True})]) + return LMResponse(model=request.model, outputs=[lm_output_from_legacy_output(output) for output in outputs]) + + +def legacy_outputs_from_lm_response(response: LMResponse) -> list[dict[str, Any] | str | None]: + """Return legacy adapter postprocess values from a normalized response. + + This is a temporary compatibility helper for current adapter postprocessing. + Prefer the exact original legacy output when it is available so compatibility + hooks that distinguish `str` from provider dictionaries keep working. + """ + outputs = [] + for output in response.outputs: + if output.metadata.get("empty_legacy_outputs"): + continue + if output.provider_output is not None: + outputs.append(output.provider_output) + elif output.text is not None and not output.reasoning_content and not output.tool_calls and not output.citations and output.logprobs is None: + outputs.append(output.text) + else: + outputs.append(output.to_output_dict()) + return outputs + + +def lm_output_from_legacy_output(output: dict[str, Any] | str | None) -> LMOutput: + """Normalize one current legacy `BaseLM` output item into an `LMOutput`.""" + if isinstance(output, str): + return LMOutput(parts=[LMTextPart(text=output)], provider_output=output) + if output is None: + return LMOutput(parts=[]) + + parts = [] + text = output.get("text") + if text: + parts.append(LMTextPart(text=text)) + reasoning = output.get("reasoning_content") + if reasoning: + parts.append(LMThinkingPart(text=str(reasoning))) + for tool_call in output.get("tool_calls") or []: + parts.append(provider_tool_call_to_part(tool_call)) + for citation in output.get("citations") or []: + parts.append(citation_to_part(citation)) + return LMOutput(parts=parts, logprobs=output.get("logprobs"), provider_output=output) diff --git a/dspy/core/types.py b/dspy/core/types.py index 06bd907db4..6ceb9eb447 100644 --- a/dspy/core/types.py +++ b/dspy/core/types.py @@ -300,7 +300,13 @@ class LMToolChoice(BaseModel): @classmethod def from_value(cls, value: Any = None, **overrides: Any) -> LMToolChoice: - data = _config_data(value, str_field="mode") + if isinstance(value, Mapping) and value.get("type") == "function" and isinstance(value.get("function"), Mapping): + # Accept OpenAI/LiteLLM-shaped tool choices for the current BaseLM + # compatibility path. Internally, DSPy represents this as requiring + # one allowed tool. + data = {"mode": "required", "allowed": [value["function"].get("name", "")]} + else: + data = _config_data(value, str_field="mode") if "parallel_tool_calls" in data and "parallel" not in data: data["parallel"] = data.pop("parallel_tool_calls") data.update({key: value for key, value in overrides.items() if value is not _MISSING}) @@ -1759,7 +1765,7 @@ def _parts_from_openai_content(content: Any) -> list[LMPart]: for item in content: item_type = item.get("type") if isinstance(item, dict) else None if item_type == "text": - parts.append(LMTextPart(text=item.get("text", ""))) + parts.append(LMTextPart(text=item.get("text", ""), metadata=item.get("metadata", {}) or {})) elif item_type == "image_url": image = item.get("image_url", {}) if not isinstance(image, dict): @@ -1924,10 +1930,12 @@ def _coerce_tool_spec(tool: Any) -> LMToolSpec: if isinstance(tool, dict): if "function" in tool: function = tool["function"] + provider_data = {key: value for key, value in tool.items() if key not in {"type", "function"}} return LMToolSpec( name=function.get("name"), description=function.get("description"), parameters=function.get("parameters", {}), + provider_data=provider_data, ) return LMToolSpec(**tool) raise TypeError(f"Cannot convert {type(tool)!r} to LMToolSpec.") diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.py index 15a35fd30a..6eaee233a4 100644 --- a/tests/adapters/test_two_step_adapter.py +++ b/tests/adapters/test_two_step_adapter.py @@ -68,6 +68,7 @@ class TestSignature(dspy.Signature): mock_main_lm = mock.MagicMock(spec=dspy.LM) mock_main_lm.return_value = ["text from main LM"] mock_main_lm.kwargs = {"temperature": 1.0} + mock_main_lm.model = "openai/gpt-4o-mini" mock_extraction_lm = mock.MagicMock(spec=dspy.LM) mock_extraction_lm.return_value = [ @@ -134,6 +135,7 @@ class TestSignature(dspy.Signature): mock_main_lm = mock.MagicMock(spec=dspy.LM) mock_main_lm.acall.return_value = ["text from main LM"] mock_main_lm.kwargs = {"temperature": 1.0} + mock_main_lm.model = "openai/gpt-4o-mini" mock_extraction_lm = mock.MagicMock(spec=dspy.LM) mock_extraction_lm.acall.return_value = [ From b2c6ca4a3f25149d387bd7456c7539ca0a11e004 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:24:11 -0400 Subject: [PATCH 10/23] refactor(adapter): introduce adapter call plan runtime --- dspy/adapters/_type_runtime.py | 124 +++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 dspy/adapters/_type_runtime.py diff --git a/dspy/adapters/_type_runtime.py b/dspy/adapters/_type_runtime.py new file mode 100644 index 0000000000..757aa3ae22 --- /dev/null +++ b/dspy/adapters/_type_runtime.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Literal + +from pydantic.fields import FieldInfo + +from dspy.core.types import LMConfig, LMMessage, LMOutput, LMPart, LMToolSpec + +if TYPE_CHECKING: + from dspy.adapters.base import Adapter + from dspy.signatures.signature import Signature + + +@dataclass(frozen=True) +class _LMCapabilities: + model: str + model_type: str + supported_params: frozenset[str] + supports_function_calling: bool + supports_response_schema: bool + supports_reasoning: bool + + +@dataclass(frozen=True) +class _CallContext: + adapter: Adapter + use_native_function_calling: bool + allow_parallel_tool_calls: bool | None + native_response_types: tuple[type[object], ...] + lm: _LMCapabilities + lm_kwargs: dict[str, object] = field(default_factory=dict) + lm_default_kwargs: dict[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True) +class _TypeRenderContext: + field_name: str + field_info: FieldInfo + signature: type[Signature] + values: dict[str, object] + adapter: Adapter + call_context: _CallContext + role: Literal["input", "output"] + + +@dataclass(frozen=True) +class _TypeParseContext: + field_name: str + field_info: FieldInfo + signature: type[Signature] + adapter: Adapter + call_context: _CallContext + parts: list[LMPart] + lm_output: LMOutput + + +_OutputParser = Callable[[_TypeParseContext], object] + + +@dataclass +class _AdapterCallPlan: + source_signature: type[Signature] + render_signature: type[Signature] + inputs: dict[str, object] + lm_kwargs: dict[str, object] + messages: list[LMMessage] = field(default_factory=list) + tools: list[LMToolSpec] = field(default_factory=list) + config: LMConfig | None = None + output_parsers: dict[str, _OutputParser] = field(default_factory=dict) + + @classmethod + def from_signature( + cls, + signature: type[Signature], + inputs: dict[str, object], + lm_kwargs: dict[str, object] | None = None, + ) -> _AdapterCallPlan: + return cls( + source_signature=signature, + render_signature=signature, + inputs=dict(inputs), + lm_kwargs=dict(lm_kwargs or {}), + ) + + def delete_field(self, field_name: str) -> None: + if field_name in self.render_signature.fields: + self.render_signature = self.render_signature.delete(field_name) + + def merge_config(self, config: LMConfig | None) -> None: + if config is None: + return + self.config = _merge_lm_config(self.config, config) + + +class _TypeFeatureHandler: + def prepare(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + pass + + def parse( + self, + values: dict[str, object], + output: LMOutput, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> None: + pass + + +def _merge_lm_config(left: LMConfig | None, right: LMConfig | None) -> LMConfig | None: + if left is None: + return right + if right is None: + return left + + data = left.model_dump() + right_data = right.model_dump(exclude_none=True, exclude_unset=True) + for key in ("reasoning", "tool_choice", "cache", "prompt_cache"): + if key in right_data and isinstance(data.get(key), dict) and isinstance(right_data[key], dict): + right_data[key] = {**data[key], **right_data[key]} + data.update(right_data) + data["extensions"] = {**left.extensions, **right.extensions} + return LMConfig(**data) From 4b00ae5cd8888557d9056818937fe87e59bee7e4 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:24:26 -0400 Subject: [PATCH 11/23] feat(adapter): add canonical tool result values --- dspy/__init__.py | 2 +- dspy/adapters/__init__.py | 14 +- dspy/adapters/types/__init__.py | 15 ++- dspy/adapters/types/tool.py | 221 ++++++++++++++++++++++++++++---- 4 files changed, 221 insertions(+), 31 deletions(-) diff --git a/dspy/__init__.py b/dspy/__init__.py index 76afdb805d..cc9d1fe5e8 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -6,7 +6,7 @@ from dspy.evaluate import Evaluate # isort: skip from dspy.clients import * # isort: skip -from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, Code, Reasoning # isort: skip +from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, ToolCallResults, Code, Reasoning # isort: skip from dspy.primitives.sandbox_serializable import SandboxSerializable # isort: skip from dspy.utils.exceptions import ContextWindowExceededError from dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_logging diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.py index c217d7260e..23a6631ebc 100644 --- a/dspy/adapters/__init__.py +++ b/dspy/adapters/__init__.py @@ -2,7 +2,18 @@ from dspy.adapters.chat_adapter import ChatAdapter from dspy.adapters.json_adapter import JSONAdapter from dspy.adapters.two_step_adapter import TwoStepAdapter -from dspy.adapters.types import Audio, Code, File, History, Image, Reasoning, Tool, ToolCalls, Type +from dspy.adapters.types import ( + Audio, + Code, + File, + History, + Image, + Reasoning, + Tool, + ToolCallResults, + ToolCalls, + Type, +) from dspy.adapters.xml_adapter import XMLAdapter __all__ = [ @@ -19,5 +30,6 @@ "TwoStepAdapter", "Tool", "ToolCalls", + "ToolCallResults", "Reasoning", ] diff --git a/dspy/adapters/types/__init__.py b/dspy/adapters/types/__init__.py index 5ec8043021..c70878dc2e 100644 --- a/dspy/adapters/types/__init__.py +++ b/dspy/adapters/types/__init__.py @@ -5,6 +5,17 @@ from dspy.adapters.types.history import History from dspy.adapters.types.image import Image from dspy.adapters.types.reasoning import Reasoning -from dspy.adapters.types.tool import Tool, ToolCalls +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls -__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "Code", "Reasoning"] +__all__ = [ + "History", + "Image", + "Audio", + "File", + "Type", + "Tool", + "ToolCalls", + "ToolCallResults", + "Code", + "Reasoning", +] diff --git a/dspy/adapters/types/tool.py b/dspy/adapters/types/tool.py index e6deb9b7c2..4ceea4e2c7 100644 --- a/dspy/adapters/types/tool.py +++ b/dspy/adapters/types/tool.py @@ -1,12 +1,13 @@ import asyncio import inspect -from typing import TYPE_CHECKING, Any, Callable, get_origin, get_type_hints +from typing import TYPE_CHECKING, Any, Callable, ClassVar, get_origin, get_type_hints import pydantic from jsonschema import ValidationError, validate from pydantic import BaseModel, TypeAdapter, create_model from dspy.adapters.types.base_type import Type +from dspy.core.types import LMMessage, LMTextPart, LMToolCallPart, LMToolResultPart, LMToolSpec from dspy.dsp.utils.settings import settings from dspy.utils.callback import with_callbacks @@ -162,6 +163,14 @@ def format_as_litellm_function_call(self): }, } + def to_lm_tool_spec(self) -> LMToolSpec: + args = self.args or {} + return LMToolSpec( + name=self.name or "", + description=self.desc, + parameters={"type": "object", "properties": args, "required": list(args.keys())}, + ) + def _run_async_in_sync(self, coroutine): try: loop = asyncio.get_running_loop() @@ -260,18 +269,21 @@ def __str__(self): class ToolCalls(Type): + JSON_SCHEMA_EXTRA_MAX_ITEMS_KEY: ClassVar[str] = "__dspy_tool_calls_max_items" + class ToolCall(Type): name: str args: dict[str, Any] + id: str | None = None def format(self): - return { - "type": "function", - "function": { - "name": self.name, - "arguments": self.args, - }, - } + formatted = {"name": self.name, "args": self.args} + if self.id is not None: + formatted["id"] = self.id + return formatted + + def to_lm_part(self, tool_call_id: str | None = None) -> LMToolCallPart: + return LMToolCallPart(id=tool_call_id or self.id, name=self.name, args=self.args) def execute(self, functions: dict[str, Any] | list[Tool] | None = None) -> Any: """Execute this individual tool call and return its result. @@ -320,6 +332,41 @@ def execute(self, functions: dict[str, Any] | list[Tool] | None = None) -> Any: tool_calls: list[ToolCall] + @classmethod + def json_schema(cls, *, max_items: int | None = None) -> dict[str, Any]: + """Return the prompt-facing JSON schema for ToolCalls.""" + if max_items is not None and max_items < 1: + raise ValueError("ToolCalls JSON schema `max_items` must be at least 1.") + + schema = pydantic.TypeAdapter(cls).json_schema() + if max_items is not None: + schema.setdefault("properties", {}).setdefault("tool_calls", {})["maxItems"] = max_items + return schema + + @classmethod + def json_schema_extra_for_max_items(cls, max_items: int | None) -> dict[str, Any]: + if max_items is None: + return {} + return {cls.JSON_SCHEMA_EXTRA_MAX_ITEMS_KEY: max_items} + + @classmethod + def max_items_from_field_info(cls, field_info: Any) -> int | None: + if field_info is None: + return None + field_extra = getattr(field_info, "json_schema_extra", None) or {} + return field_extra.get(cls.JSON_SCHEMA_EXTRA_MAX_ITEMS_KEY) + + def validate_max_items(self, max_items: int | None) -> None: + if max_items is None: + return + if len(self.tool_calls) <= max_items: + return + raise ValueError( + f"`dspy.ToolCalls` received {len(self.tool_calls)} tool calls, but this call permits at most " + f"{max_items}. Set `allow_parallel_tool_calls=True` to accept multiple tool calls in one LM turn, " + "or return one tool call per turn." + ) + @classmethod def from_dict_list(cls, tool_calls_dicts: list[dict[str, Any]]) -> "ToolCalls": """Convert a list of dictionaries to a ToolCalls instance. @@ -340,8 +387,7 @@ def from_dict_list(cls, tool_calls_dicts: list[dict[str, Any]]) -> "ToolCalls": tool_calls = ToolCalls.from_dict_list(tool_calls_dict) ``` """ - tool_calls = [cls.ToolCall(**item) for item in tool_calls_dicts] - return cls(tool_calls=tool_calls) + return cls.model_validate(tool_calls_dicts) @classmethod def description(cls) -> str: @@ -350,41 +396,162 @@ def description(cls) -> str: "Arguments must be provided in JSON format." ) - def format(self) -> list[dict[str, Any]]: - # The tool_call field is compatible with OpenAI's tool calls schema. + def format(self) -> dict[str, Any]: return { "tool_calls": [tool_call.format() for tool_call in self.tool_calls], } + @classmethod + def parse_lm_response(cls, response: str | dict[str, Any]) -> "ToolCalls | None": + if not isinstance(response, dict): + return None + tool_calls = response.get("tool_calls") + if not tool_calls: + return None + return cls.model_validate(tool_calls) + + def to_lm_parts(self, id_prefix: str | None = None) -> list[LMToolCallPart]: + return [ + tool_call.to_lm_part(f"{id_prefix}_{idx}" if id_prefix is not None and tool_call.id is None else None) + for idx, tool_call in enumerate(self.tool_calls) + ] + + def with_call_ids(self, id_prefix: str) -> "ToolCalls": + tool_calls = [ + tool_call if tool_call.id is not None else tool_call.model_copy(update={"id": f"{id_prefix}_{idx}"}) + for idx, tool_call in enumerate(self.tool_calls) + ] + return self.model_copy(update={"tool_calls": tool_calls}) + + @classmethod + def _canonical_tool_call(cls, item: Any) -> Any: + if isinstance(item, cls.ToolCall): + return item.model_dump() + if isinstance(item, dict): + if "name" in item and "args" in item: + normalized = {"name": item["name"], "args": item["args"]} + if "id" in item: + normalized["id"] = item["id"] + return normalized + return item + if hasattr(item, "name") and hasattr(item, "args"): + normalized = {"name": item.name, "args": item.args} + tool_call_id = getattr(item, "id", None) + if tool_call_id is not None: + normalized["id"] = tool_call_id + return normalized + return item + @pydantic.model_validator(mode="before") @classmethod def validate_input(cls, data: Any): if isinstance(data, cls): return data - # Handle case where data is a list of dicts with "name" and "args" keys - if isinstance(data, list) and all( - isinstance(item, dict) and "name" in item and "args" in item for item in data - ): - return {"tool_calls": [cls.ToolCall(**item) for item in data]} - # Handle case where data is a dict + tool_calls_data = None + if isinstance(data, list): + tool_calls_data = data elif isinstance(data, dict): if "tool_calls" in data: - # Handle case where data is a dict with "tool_calls" key tool_calls_data = data["tool_calls"] - if isinstance(tool_calls_data, list): - return { - "tool_calls": [ - cls.ToolCall(**item) if isinstance(item, dict) else item for item in tool_calls_data - ] - } elif "name" in data and "args" in data: - # Handle case where data is a dict with "name" and "args" keys - return {"tool_calls": [cls.ToolCall(**data)]} + return {"tool_calls": [cls._canonical_tool_call(data)]} + + if isinstance(tool_calls_data, list): + normalized = [cls._canonical_tool_call(item) for item in tool_calls_data] + if all(isinstance(item, dict) and "name" in item and "args" in item for item in normalized): + return {"tool_calls": normalized} raise ValueError(f"Received invalid value for `dspy.ToolCalls`: {data}") +class _ToolCallResult(BaseModel): + call_id: str | None = None + name: str | None = None + value: Any + is_error: bool = False + + @staticmethod + def _format_value(value: Any) -> str: + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return str(value) + + def to_lm_message(self) -> LMMessage: + return LMMessage( + role="tool", + parts=[ + LMToolResultPart( + call_id=self.call_id, + name=self.name, + content=[LMTextPart(text=self._format_value(self.value))], + is_error=self.is_error, + ) + ], + ) + + def format(self) -> dict[str, Any]: + formatted = {"value": self.value, "is_error": self.is_error} + if self.call_id is not None: + formatted["call_id"] = self.call_id + if self.name is not None: + formatted["name"] = self.name + return formatted + + +class ToolCallResults(Type): + tool_call_results: list[_ToolCallResult] + + @classmethod + def from_dict_list(cls, tool_call_result_dicts: list[dict[str, Any]]) -> "ToolCallResults": + return cls.model_validate(tool_call_result_dicts) + + @classmethod + def from_tool_calls_and_values( + cls, + tool_calls: list[ToolCalls.ToolCall], + values: list[Any], + is_errors: list[bool], + ) -> "ToolCallResults": + return cls( + tool_call_results=[ + _ToolCallResult( + call_id=tool_call.id, + name=tool_call.name, + value=value, + is_error=is_error, + ) + for tool_call, value, is_error in zip(tool_calls, values, is_errors, strict=True) + ] + ) + + @classmethod + def description(cls) -> str: + return ( + "Tool call results, including the tool call id, tool name, returned value, and whether the tool " + "execution failed." + ) + + def format(self) -> dict[str, Any]: + return {"tool_call_results": [result.format() for result in self.tool_call_results]} + + def to_lm_messages(self) -> list[LMMessage]: + return [result.to_lm_message() for result in self.tool_call_results] + + @pydantic.model_validator(mode="before") + @classmethod + def validate_input(cls, data: Any): + if isinstance(data, cls): + return data + if isinstance(data, list): + return {"tool_call_results": data} + if isinstance(data, dict) and "tool_call_results" in data: + return data + if isinstance(data, dict) and "value" in data: + return {"tool_call_results": [data]} + raise ValueError(f"Received invalid value for `dspy.ToolCallResults`: {data}") + + def _resolve_json_schema_reference(schema: dict) -> dict: """Recursively resolve json model schema, expanding all references.""" From 81765e6e8c58a04dbb52121c89e6665acf6f6245 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:24:43 -0400 Subject: [PATCH 12/23] refactor(adapter): move type rendering to default hooks --- docs/docs/api/experimental/Citations.md | 1 - docs/docs/api/experimental/Document.md | 1 - docs/docs/api/primitives/Audio.md | 1 - docs/docs/api/primitives/Code.md | 1 - docs/docs/api/primitives/Image.md | 1 - docs/docs/api/primitives/Tool.md | 1 - docs/docs/api/primitives/ToolCalls.md | 1 - dspy/adapters/types/base_type.py | 47 +++++++++---------------- dspy/adapters/types/citation.py | 6 ---- dspy/adapters/types/reasoning.py | 40 +-------------------- 10 files changed, 18 insertions(+), 82 deletions(-) diff --git a/docs/docs/api/experimental/Citations.md b/docs/docs/api/experimental/Citations.md index 3738fd8922..a123680e99 100644 --- a/docs/docs/api/experimental/Citations.md +++ b/docs/docs/api/experimental/Citations.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/experimental/Document.md b/docs/docs/api/experimental/Document.md index 5b6fb6d643..3439461878 100644 --- a/docs/docs/api/experimental/Document.md +++ b/docs/docs/api/experimental/Document.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/Audio.md b/docs/docs/api/primitives/Audio.md index 129d9b358f..de097d709c 100644 --- a/docs/docs/api/primitives/Audio.md +++ b/docs/docs/api/primitives/Audio.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/Code.md b/docs/docs/api/primitives/Code.md index d84b4b05ad..9dacbd9486 100644 --- a/docs/docs/api/primitives/Code.md +++ b/docs/docs/api/primitives/Code.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/Image.md b/docs/docs/api/primitives/Image.md index fc46ce9cc7..924ad46ecb 100644 --- a/docs/docs/api/primitives/Image.md +++ b/docs/docs/api/primitives/Image.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/Tool.md b/docs/docs/api/primitives/Tool.md index 45f447e50c..3c58364743 100644 --- a/docs/docs/api/primitives/Tool.md +++ b/docs/docs/api/primitives/Tool.md @@ -7,7 +7,6 @@ members: - __call__ - acall - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/ToolCalls.md b/docs/docs/api/primitives/ToolCalls.md index 350bf0a9c8..5394ebb3ae 100644 --- a/docs/docs/api/primitives/ToolCalls.md +++ b/docs/docs/api/primitives/ToolCalls.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/dspy/adapters/types/base_type.py b/dspy/adapters/types/base_type.py index 13a55727f2..2a629559b4 100644 --- a/dspy/adapters/types/base_type.py +++ b/dspy/adapters/types/base_type.py @@ -5,13 +5,12 @@ import json_repair import pydantic -from dspy.clients.base_lm import BaseLM +from dspy.adapters._type_runtime import _TypeParseContext, _TypeRenderContext +from dspy.core.types import LMPart if TYPE_CHECKING: from litellm import ModelResponseStream - from dspy.signatures.signature import Signature - CUSTOM_TYPE_START_IDENTIFIER = "<>" CUSTOM_TYPE_END_IDENTIFIER = "<>" @@ -36,6 +35,16 @@ def format(self) -> list[dict[str, Any]]: def format(self) -> list[dict[str, Any]] | str: raise NotImplementedError + def default_render_input(self, ctx: _TypeRenderContext) -> list[LMPart]: + return ctx.adapter.value_to_lm_parts(self, ctx.field_info) + + def default_render_output(self, ctx: _TypeRenderContext) -> list[LMPart]: + return ctx.adapter.value_to_lm_parts(self, ctx.field_info) + + @classmethod + def default_parse_output(cls, ctx: _TypeParseContext) -> object: + return ctx.adapter.lm_parts_to_value(ctx.parts, ctx.field_info) + @classmethod def description(cls) -> str: """Description of the custom type""" @@ -76,31 +85,6 @@ def serialize_model(self): ) return formatted - @classmethod - def adapt_to_native_lm_feature( - cls, - signature: type["Signature"], - field_name: str, - lm: BaseLM, - lm_kwargs: dict[str, Any], - ) -> type["Signature"]: - """Adapt the custom type to the native LM feature if possible. - - When the LM and configuration supports the related native LM feature, e.g., native tool calling, native - reasoning, etc., we adapt the signature and `lm_kwargs` to enable the native LM feature. - - Args: - signature: The DSPy signature for the LM call. - field_name: The name of the field in the signature to adapt to the native LM feature. - lm: The LM instance. - lm_kwargs: The keyword arguments for the LM call, subject to in-place updates if adaptation if required. - - Returns: - The adapted signature. If the custom type is not natively supported by the LM, return the original - signature. - """ - return signature - @classmethod def is_streamable(cls) -> bool: """Whether the custom type is streamable.""" @@ -166,8 +150,11 @@ def split_message_content_for_custom_types(messages: list[dict[str, Any]]) -> li pattern = rf"{CUSTOM_TYPE_START_IDENTIFIER}(.*?){CUSTOM_TYPE_END_IDENTIFIER}" result = [] last_end = 0 - # DSPy adapter always formats user input into a string content before custom type splitting - content: str = message["content"] + # Normalized adapter paths may have already rendered custom values into + # content blocks. The legacy marker splitter only applies to text. + content = message["content"] + if not isinstance(content, str): + continue for match in re.finditer(pattern, content, re.DOTALL): start, end = match.span() diff --git a/dspy/adapters/types/citation.py b/dspy/adapters/types/citation.py index a8c5461a58..b6640968cb 100644 --- a/dspy/adapters/types/citation.py +++ b/dspy/adapters/types/citation.py @@ -167,12 +167,6 @@ def __getitem__(self, index): """Allow indexing into citations.""" return self.citations[index] - @classmethod - def adapt_to_native_lm_feature(cls, signature, field_name, lm, lm_kwargs) -> bool: - if lm.model.startswith("anthropic/"): - return signature.delete(field_name) - return signature - @classmethod def is_streamable(cls) -> bool: """Whether the Citations type is streamable.""" diff --git a/dspy/adapters/types/reasoning.py b/dspy/adapters/types/reasoning.py index c4fa3de742..08f229348f 100644 --- a/dspy/adapters/types/reasoning.py +++ b/dspy/adapters/types/reasoning.py @@ -1,12 +1,8 @@ -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional import pydantic from dspy.adapters.types.base_type import Type -from dspy.clients.base_lm import BaseLM - -if TYPE_CHECKING: - from dspy.signatures.signature import Signature class Reasoning(Type): @@ -42,40 +38,6 @@ def validate_input(cls, data: Any): raise ValueError(f"Received invalid value for `dspy.Reasoning`: {data}") - @classmethod - def adapt_to_native_lm_feature( - cls, - signature: type["Signature"], - field_name: str, - lm: BaseLM, - lm_kwargs: dict[str, Any], - ) -> type["Signature"]: - if "reasoning_effort" in lm_kwargs: - # `lm_kwargs` overrides `lm.kwargs`. - reasoning_effort = lm_kwargs["reasoning_effort"] - elif "reasoning_effort" in lm.kwargs: - reasoning_effort = lm.kwargs["reasoning_effort"] - else: - # Turn on the native reasoning explicitly if Reasoning field is present in the signature and no explicit - # reasoning effort is set in `lm_kwargs` or `lm.kwargs`. - reasoning_effort = "low" - - if reasoning_effort is None or not lm.supports_reasoning: - # If users explicitly set `reasoning_effort` to None or the LM doesn't support reasoning, we don't enable - # native reasoning. - return signature - - if "gpt-5" in lm.model and lm.model_type == "chat": - # There is a caveat of Litellm as 1.79.0 that when using the chat completion API on GPT-5 family models, - # the reasoning content is not available in the response. As a workaround, we don't enable the native - # reasoning feature for GPT-5 family models when using the chat completion API. - # Litellm issue: https://github.com/BerriAI/litellm/issues/14748 - return signature - - lm_kwargs["reasoning_effort"] = reasoning_effort - # Delete the reasoning field from the signature to use the native reasoning feature. - return signature.delete(field_name) - @classmethod def parse_lm_response(cls, response: str | dict[str, Any]) -> Optional["Reasoning"]: """Parse the LM response into a Reasoning object.""" From c3bf0c5d5124cce55018c31fe13e3a6c0c7bc526 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:24:57 -0400 Subject: [PATCH 13/23] feat(adapter): add type feature handlers --- dspy/adapters/_type_feature_handlers.py | 267 ++++++++++++++++++ tests/adapters/test_adapter_base.py | 63 +++++ .../test_adapter_migration_robustness.py | 83 ++++++ tests/adapters/test_tool.py | 222 ++++++++++++++- 4 files changed, 620 insertions(+), 15 deletions(-) create mode 100644 dspy/adapters/_type_feature_handlers.py create mode 100644 tests/adapters/test_adapter_base.py create mode 100644 tests/adapters/test_adapter_migration_robustness.py diff --git a/dspy/adapters/_type_feature_handlers.py b/dspy/adapters/_type_feature_handlers.py new file mode 100644 index 0000000000..097c4bb76a --- /dev/null +++ b/dspy/adapters/_type_feature_handlers.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, get_args + +from dspy.adapters._type_runtime import _AdapterCallPlan, _CallContext, _TypeFeatureHandler +from dspy.adapters.types.reasoning import Reasoning +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls +from dspy.core.types import ( + LMCitationPart, + LMConfig, + LMMessage, + LMOutput, + LMPart, + LMReasoningConfig, + LMThinkingPart, + LMToolChoice, +) +from dspy.experimental import Citations + + +@dataclass +class _RenderedTypeOutput: + parts: list[LMPart] = field(default_factory=list) + messages: list[LMMessage] = field(default_factory=list) + + @property + def consumed(self) -> bool: + return bool(self.parts or self.messages) + + +class _ToolTypeHandler(_TypeFeatureHandler): + def prepare(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + tool_call_output_field_name = _tool_call_output_field_name(call.source_signature) + tool_input_field_name = _tool_input_field_name(call.source_signature) + native = _native_tool_calling_enabled(call, ctx) + + if ctx.use_native_function_calling and tool_call_output_field_name is not None and tool_input_field_name is None: + raise ValueError( + f"You provided an output field {tool_call_output_field_name} to receive the tool calls information, " + "but did not provide any tools as the input. Please provide a list of tools as the input by adding an " + "input field with type `list[dspy.Tool]`." + ) + + if not native: + self._apply_text_parallel_policy(call, ctx) + return + + if ctx.allow_parallel_tool_calls is not None: + call.merge_config(LMConfig(tool_choice=LMToolChoice(parallel=ctx.allow_parallel_tool_calls))) + + if tool_input_field_name is not None: + value = call.inputs.get(tool_input_field_name) + if value is not None: + for tool in value if isinstance(value, list) else [value]: + tool = tool if isinstance(tool, Tool) else Tool.model_validate(tool) + call.tools.append(tool.to_lm_tool_spec()) + call.inputs.pop(tool_input_field_name, None) + call.delete_field(tool_input_field_name) + + if tool_call_output_field_name is not None: + call.delete_field(tool_call_output_field_name) + + for field_name, field_info in call.source_signature.input_fields.items(): + if not _annotation_includes(field_info.annotation, ToolCallResults): + continue + value = call.inputs.get(field_name) + if value is None: + continue + results = value if isinstance(value, ToolCallResults) else ToolCallResults.model_validate(value) + call.messages.extend(results.to_lm_messages()) + call.inputs.pop(field_name, None) + call.delete_field(field_name) + + def parse( + self, + values: dict[str, object], + output: LMOutput, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> None: + if not _native_tool_calling_enabled(call, ctx): + return + field_name = _tool_call_output_field_name(call.source_signature) + if field_name is None or field_name in values or not output.tool_calls: + return + values[field_name] = ToolCalls.from_dict_list( + [{"name": tool_call.name, "args": tool_call.args, "id": tool_call.id} for tool_call in output.tool_calls] + ) + + def format_output( + self, + field_name: str, + value: object, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> _RenderedTypeOutput | None: + if not _native_tool_calling_enabled(call, ctx): + return None + if field_name != _tool_call_output_field_name(call.source_signature): + return None + tool_calls = value if isinstance(value, ToolCalls) else ToolCalls.model_validate(value) + return _RenderedTypeOutput(parts=list(tool_calls.to_lm_parts())) + + def _apply_text_parallel_policy(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + if ctx.allow_parallel_tool_calls is not False: + return + field_name = _tool_call_output_field_name(call.render_signature) + if field_name is None: + return + call.render_signature = call.render_signature.with_updated_fields( + field_name, + **ToolCalls.json_schema_extra_for_max_items(1), + ) + + +class _ReasoningTypeHandler(_TypeFeatureHandler): + def prepare(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + reasoning_effort = self._native_reasoning_effort(call, ctx) + if reasoning_effort is None: + return + for field_name, field_info in call.source_signature.output_fields.items(): + if field_info.annotation is Reasoning: + call.merge_config(LMConfig(reasoning=LMReasoningConfig(effort=reasoning_effort))) + call.delete_field(field_name) + + def parse( + self, + values: dict[str, object], + output: LMOutput, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> None: + if self._native_reasoning_effort(call, ctx) is None or not output.reasoning_content: + return + for field_name, field_info in call.source_signature.output_fields.items(): + if field_info.annotation is Reasoning and field_name not in values: + values[field_name] = Reasoning(content=output.reasoning_content) + + def format_output( + self, + field_name: str, + value: object, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> _RenderedTypeOutput | None: + if self._native_reasoning_effort(call, ctx) is None: + return None + field_info = call.source_signature.output_fields.get(field_name) + if field_info is None or field_info.annotation is not Reasoning: + return None + reasoning = value if isinstance(value, Reasoning) else Reasoning.model_validate(value) + return _RenderedTypeOutput(parts=[LMThinkingPart(text=reasoning.content)]) + + @staticmethod + def _native_reasoning_effort(call: _AdapterCallPlan, ctx: _CallContext) -> str | None: + if Reasoning not in ctx.native_response_types: + return None + if not ctx.lm.supports_reasoning: + return None + if "gpt-5" in ctx.lm.model and ctx.lm.model_type == "chat": + return None + + if "reasoning_effort" in call.lm_kwargs: + reasoning_effort = call.lm_kwargs["reasoning_effort"] + elif "reasoning_effort" in ctx.lm_default_kwargs: + reasoning_effort = ctx.lm_default_kwargs["reasoning_effort"] + else: + reasoning_effort = "low" + return reasoning_effort + + +class _CitationsTypeHandler(_TypeFeatureHandler): + def prepare(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + if not self._native_citations_enabled(ctx): + return + for field_name, field_info in call.source_signature.output_fields.items(): + if field_info.annotation is Citations: + call.delete_field(field_name) + + def parse( + self, + values: dict[str, object], + output: LMOutput, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> None: + if not self._native_citations_enabled(ctx) or not output.citations: + return + for field_name, field_info in call.source_signature.output_fields.items(): + if field_info.annotation is Citations and field_name not in values: + values[field_name] = Citations.from_dict_list( + [_lm_part_to_citation_dict(citation) for citation in output.citations] + ) + + def format_output( + self, + field_name: str, + value: object, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> _RenderedTypeOutput | None: + if not self._native_citations_enabled(ctx): + return None + field_info = call.source_signature.output_fields.get(field_name) + if field_info is None or field_info.annotation is not Citations: + return None + citations = value if isinstance(value, Citations) else Citations.model_validate(value) + return _RenderedTypeOutput(parts=[_citation_to_lm_part(citation) for citation in citations.citations]) + + @staticmethod + def _native_citations_enabled(ctx: _CallContext) -> bool: + return Citations in ctx.native_response_types and ctx.lm.model.startswith("anthropic/") + + +def _native_tool_calling_enabled(call: _AdapterCallPlan, ctx: _CallContext) -> bool: + return bool( + ctx.use_native_function_calling + and ctx.lm.supports_function_calling + and _tool_call_output_field_name(call.source_signature) is not None + ) + + +def _tool_input_field_name(signature: type[Any]) -> str | None: + for name, field in signature.input_fields.items(): + if _annotation_includes(field.annotation, Tool): + return name + return None + + +def _tool_call_output_field_name(signature: type[Any]) -> str | None: + for name, field in signature.output_fields.items(): + if _annotation_includes(field.annotation, ToolCalls): + return name + return None + + +def _annotation_includes(annotation: Any, target: type) -> bool: + if annotation is target: + return True + return any(_annotation_includes(arg, target) for arg in get_args(annotation)) + + +def _citation_to_lm_part(citation: Citations.Citation) -> LMCitationPart: + return LMCitationPart( + text=citation.cited_text, + title=citation.document_title, + metadata={ + "type": citation.type, + "document_index": citation.document_index, + "start_char_index": citation.start_char_index, + "end_char_index": citation.end_char_index, + **({"supported_text": citation.supported_text} if citation.supported_text is not None else {}), + }, + ) + + +def _lm_part_to_citation_dict(citation: LMCitationPart) -> dict[str, Any]: + data = dict(citation.metadata) + if citation.text is not None: + data["cited_text"] = citation.text + if citation.title is not None: + data["document_title"] = citation.title + data.setdefault("document_index", 0) + data.setdefault("start_char_index", 0) + data.setdefault("end_char_index", len(data.get("cited_text", ""))) + return data diff --git a/tests/adapters/test_adapter_base.py b/tests/adapters/test_adapter_base.py new file mode 100644 index 0000000000..a7cbf920a7 --- /dev/null +++ b/tests/adapters/test_adapter_base.py @@ -0,0 +1,63 @@ +import dspy +from dspy.core.types import LMConfig, LMToolChoice +from tests.adapters.conftest import format_messages_and_lm_kwargs + + +def test_adapter_does_not_require_native_tool_call_format_kwarg_when_unused(): + class QA(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + class OldStyleFormatAdapter(dspy.Adapter): + def format(self, signature, demos, inputs): + return [{"role": "user", "content": inputs["question"]}] + + def parse(self, signature, completion): + return {"answer": completion} + + messages, lm_kwargs = format_messages_and_lm_kwargs( + OldStyleFormatAdapter(), + QA, + [], + {"question": "Q"}, + ) + + assert messages == [{"role": "user", "content": "Q"}] + assert lm_kwargs == {} + + +def test_call_context_does_not_carry_mutable_extra_state(): + context = dspy.Adapter(allow_parallel_tool_calls=False).build_call_context( + dspy.utils.DummyLM([{}]), + {"temperature": 0.2}, + ) + + assert not hasattr(context, "extra") + assert context.allow_parallel_tool_calls is False + assert context.lm_kwargs == {"temperature": 0.2} + assert context.lm_default_kwargs["temperature"] == 0.0 + + +def test_adapter_merges_lm_config_patches_without_losing_nested_fields(): + adapter = dspy.Adapter() + left = LMConfig.from_kwargs(reasoning={"effort": "low"}, cache=True, custom="left") + right = LMConfig.from_kwargs(reasoning={"max_tokens": 64}, rollout_id="rollout", other="right") + + merged = adapter._merge_lm_config(left, right) + + assert merged.reasoning.effort == "low" + assert merged.reasoning.max_tokens == 64 + assert merged.cache.enabled is True + assert merged.cache.rollout_id == "rollout" + assert merged.extensions == {"custom": "left", "other": "right"} + + +def test_adapter_lm_config_patch_preserves_explicit_tool_choice_mode(): + adapter = dspy.Adapter() + merged_kwargs = adapter._merge_config_kwargs( + {"tool_choice": "none"}, + LMConfig(tool_choice=LMToolChoice(parallel=False)), + ) + + assert merged_kwargs["tool_choice"]["mode"] == "none" + assert merged_kwargs["tool_choice"]["parallel"] is False diff --git a/tests/adapters/test_adapter_migration_robustness.py b/tests/adapters/test_adapter_migration_robustness.py new file mode 100644 index 0000000000..a391defd07 --- /dev/null +++ b/tests/adapters/test_adapter_migration_robustness.py @@ -0,0 +1,83 @@ +import dspy +from dspy.utils.exceptions import AdapterParseError + + +def add(a: int, b: int) -> int: + return a + b + + +class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + def __init__(self, output): + self.output = output + self.messages = None + self.kwargs = None + + def __call__(self, messages, **kwargs): + self.messages = messages + self.kwargs = kwargs + return [self.output] + + +def test_native_tool_response_can_combine_visible_text_and_tool_calls(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + answer: str = dspy.OutputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + lm = NativeToolLM( + { + "text": "[[ ## answer ## ]]\nworking\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "id": "call_add", + "type": "function", + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + } + ], + } + ) + + result = dspy.ChatAdapter(use_native_function_calling=True)( + lm, + {}, + ToolSignature, + [], + {"question": "What is 1+2?", "tools": [dspy.Tool(add)]}, + )[0] + + assert result["answer"] == "working" + assert result["tool_calls"].tool_calls[0].id == "call_add" + assert result["tool_calls"].tool_calls[0].args == {"a": 1, "b": 2} + + +def test_tool_type_handler_does_not_override_explicit_tool_choice_mode(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + lm = NativeToolLM({"text": None}) + adapter = dspy.ChatAdapter(use_native_function_calling=True, allow_parallel_tool_calls=False) + + try: + adapter( + lm, + {"tool_choice": "none"}, + ToolSignature, + [], + {"question": "What is 1+2?", "tools": [dspy.Tool(add)]}, + ) + except AdapterParseError: + pass + + assert lm.kwargs["tool_choice"] == "none" + assert lm.kwargs["parallel_tool_calls"] is False + assert lm.kwargs["tools"][0]["function"]["parameters"]["properties"]["a"]["type"] == "integer" diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py index cfcffe0947..e0f726dc83 100644 --- a/tests/adapters/test_tool.py +++ b/tests/adapters/test_tool.py @@ -5,7 +5,10 @@ from pydantic import BaseModel import dspy -from dspy.adapters.types.tool import Tool, ToolCalls, convert_input_schema_to_tool_args +from dspy.adapters._type_feature_handlers import _ToolTypeHandler +from dspy.adapters._type_runtime import _AdapterCallPlan +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls, convert_input_schema_to_tool_args +from dspy.core.types import LMOutput, LMToolCallPart # Test fixtures @@ -397,9 +400,7 @@ def test_async_tool_call_in_sync_mode(): ([], {"tool_calls": []}), ( [{"name": "search", "args": {"query": "hello"}}], - { - "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}], - }, + {"tool_calls": [{"name": "search", "args": {"query": "hello"}}]}, ), ( [ @@ -408,19 +409,14 @@ def test_async_tool_call_in_sync_mode(): ], { "tool_calls": [ - {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}, - { - "type": "function", - "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}}, - }, + {"name": "search", "args": {"query": "hello"}}, + {"name": "translate", "args": {"text": "world", "lang": "fr"}}, ], }, ), ( [{"name": "get_time", "args": {}}], - { - "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}], - }, + {"tool_calls": [{"name": "get_time", "args": {}}]}, ), ] @@ -446,8 +442,163 @@ def test_tool_calls_format_from_dict_list(): result = tool_calls.format() assert len(result["tool_calls"]) == 2 - assert result["tool_calls"][0]["function"]["name"] == "search" - assert result["tool_calls"][1]["function"]["name"] == "translate" + assert result["tool_calls"][0]["name"] == "search" + assert result["tool_calls"][1]["name"] == "translate" + + +def test_tool_calls_preserve_call_ids_and_fill_missing_ids(): + tool_calls = ToolCalls.from_dict_list( + [ + {"name": "search", "args": {"query": "hello"}, "id": "call_search"}, + {"name": "lookup", "args": {"key": "world"}}, + ] + ).with_call_ids("call") + + assert [tool_call.id for tool_call in tool_calls.tool_calls] == ["call_search", "call_1"] + assert [part.id for part in tool_calls.to_lm_parts()] == ["call_search", "call_1"] + + +def test_tool_calls_json_schema_can_limit_cardinality(): + schema = ToolCalls.json_schema(max_items=1) + + assert schema["properties"]["tool_calls"]["maxItems"] == 1 + + +def test_tool_calls_validate_max_items_rejects_too_many_calls(): + tool_calls = ToolCalls.from_dict_list( + [ + {"name": "search", "args": {"query": "hello"}}, + {"name": "lookup", "args": {"key": "world"}}, + ] + ) + + with pytest.raises(ValueError, match="permits at most 1"): + tool_calls.validate_max_items(1) + + +def test_native_tool_response_preserves_call_ids(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + answer: str = dspy.OutputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class ToolCallLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + + def __call__(self, messages, **kwargs): + return [ + { + "text": None, + "tool_calls": [ + { + "function": {"arguments": '{"x":1,"y":"two"}', "name": "dummy_function"}, + "id": "call_dummy", + "type": "function", + } + ], + } + ] + + adapter = dspy.ChatAdapter(use_native_function_calling=True) + result = adapter(ToolCallLM(), {}, ToolSignature, [], {"question": "call it", "tools": [Tool(dummy_function)]})[0] + + assert result["answer"] is None + assert result["tool_calls"] == ToolCalls.from_dict_list( + [{"name": "dummy_function", "args": {"x": 1, "y": "two"}, "id": "call_dummy"}] + ) + + +def test_tool_type_handler_renders_native_tool_specs_and_policy(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + adapter = dspy.Adapter(use_native_function_calling=True, allow_parallel_tool_calls=False) + context = adapter.build_call_context(NativeToolLM()) + call = _AdapterCallPlan.from_signature( + ToolSignature, + {"question": "Q?", "tools": [Tool(dummy_function)]}, + {}, + ) + + _ToolTypeHandler().prepare(call, context) + + assert "tools" not in call.render_signature.input_fields + assert "tool_calls" not in call.render_signature.output_fields + assert call.config.tool_choice.parallel is False + assert call.tools[0].name == "dummy_function" + + +def test_toolcalls_parse_output_reads_native_lm_tool_calls(): + class ToolSignature(dspy.Signature): + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + adapter = dspy.Adapter(use_native_function_calling=True) + context = adapter.build_call_context(NativeToolLM()) + call = _AdapterCallPlan.from_signature(ToolSignature, {"tools": [Tool(dummy_function)]}, {}) + type_handler = _ToolTypeHandler() + type_handler.prepare(call, context) + output = LMOutput(parts=[LMToolCallPart(id="call_1", name="dummy_function", args={"x": 1, "y": "two"})]) + values = {} + + type_handler.parse(values, output, call, context) + + assert values["tool_calls"] == ToolCalls.from_dict_list( + [{"name": "dummy_function", "args": {"x": 1, "y": "two"}, "id": "call_1"}] + ) + + +def test_tool_result_type_hooks_render_native_tool_messages(): + class ToolResultSignature(dspy.Signature): + tools: list[dspy.Tool] = dspy.InputField() + result: dspy.ToolCallResults = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + adapter = dspy.Adapter(use_native_function_calling=True) + result = ToolCallResults.from_dict_list([{"call_id": "call_1", "name": "dummy_function", "value": "ok"}]) + context = adapter.build_call_context(NativeToolLM()) + call = _AdapterCallPlan.from_signature( + ToolResultSignature, + {"tools": [Tool(dummy_function)], "result": result}, + {}, + ) + + _ToolTypeHandler().prepare(call, context) + + assert "result" not in call.render_signature.input_fields + assert call.messages[0].role == "tool" + assert call.messages[0].parts[0].call_id == "call_1" def test_toolcalls_vague_match(): @@ -497,6 +648,47 @@ def test_toolcalls_vague_match(): ToolCalls.model_validate([{"foo": "bar"}]) +def test_toolcalls_accepts_normalized_lm_tool_call_parts(): + tool_calls = ToolCalls.model_validate([LMToolCallPart(id="call_1", name="search", args={"query": "hello"})]) + + assert tool_calls == ToolCalls.from_dict_list([{"id": "call_1", "name": "search", "args": {"query": "hello"}}]) + + +def test_toolcalls_rejects_provider_native_shapes(): + with pytest.raises(ValueError): + ToolCalls.model_validate( + [ + { + "type": "function", + "function": {"name": "search", "arguments": '{"query": "hello"}'}, + "id": "call_1", + } + ] + ) + + with pytest.raises(ValueError): + ToolCalls.model_validate({"type": "function_call", "name": "search", "arguments": '{"query": "hello"}'}) + + +def test_tool_call_results_use_canonical_shape(): + results = ToolCallResults.model_validate( + [{"call_id": "call_1", "name": "search", "value": {"answer": "hello"}, "is_error": False}] + ) + + assert results.format() == { + "tool_call_results": [ + {"call_id": "call_1", "name": "search", "value": {"answer": "hello"}, "is_error": False} + ] + } + + +def test_tool_call_results_reject_provider_native_shapes(): + with pytest.raises(ValueError): + ToolCallResults.model_validate( + {"type": "function_call_output", "call_id": "call_1", "output": '{"answer": "hello"}'} + ) + + def test_tool_convert_input_schema_to_tool_args_no_input_params(): args, arg_types, arg_desc = convert_input_schema_to_tool_args(schema={"properties": {}}) assert args == {} @@ -577,7 +769,7 @@ def get_pi(): tool_call4 = dspy.ToolCalls.ToolCall(name="nonexistent", args={}) try: tool_call4.execute(functions=tools) - assert False, "Should have raised ValueError" + raise AssertionError("Should have raised ValueError") except ValueError as e: assert "not found" in str(e) From f5d24d644cae9665721451644dcc4c06287779fd Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:27:59 -0400 Subject: [PATCH 14/23] refactor(adapter): add normalized render helpers --- dspy/adapters/base.py | 335 +++++++++++++++++++++++++----------------- 1 file changed, 204 insertions(+), 131 deletions(-) diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index efab5c29b3..f38364de64 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -1,23 +1,43 @@ +import inspect import json import logging -from typing import Any, get_origin +from dataclasses import dataclass +from typing import Any, get_args -import json_repair +from pydantic.fields import FieldInfo from dspy.adapters._legacy_type_markers import ( - _expand_legacy_custom_type_markers_in_chat_message, _expand_legacy_custom_type_markers_in_lm_message, + _split_legacy_custom_type_text_to_parts, +) +from dspy.adapters._type_feature_handlers import ( + _CitationsTypeHandler, + _ReasoningTypeHandler, + _RenderedTypeOutput, + _ToolTypeHandler, +) +from dspy.adapters._type_runtime import ( + _AdapterCallPlan, + _CallContext, + _LMCapabilities, + _OutputParser, + _TypeFeatureHandler, + _TypeParseContext, + _TypeRenderContext, + _merge_lm_config, ) from dspy.adapters.types import History, Type from dspy.adapters.types.reasoning import Reasoning -from dspy.adapters.types.tool import Tool, ToolCalls +from dspy.adapters.types.tool import ToolCallResults, ToolCalls +from dspy.adapters.utils import format_field_value, parse_value from dspy.clients.base_lm import BaseLM from dspy.clients.openai_format import ( - legacy_outputs_from_lm_response, lm_response_from_legacy_outputs, + message_to_openai_chat, + parts_to_openai_content, to_openai_chat_request, ) -from dspy.core.types import LMMessage, LMRequest, LMResponse +from dspy.core.types import LMConfig, LMMessage, LMOutput, LMPart, LMRequest, LMResponse, LMTextPart from dspy.experimental import Citations from dspy.signatures.signature import Signature from dspy.utils.callback import BaseCallback, with_callbacks @@ -28,6 +48,17 @@ _DEFAULT_NATIVE_RESPONSE_TYPES = [Citations, Reasoning] +@dataclass +class _RenderedAdapterRequest: + request: LMRequest + call_plan: _AdapterCallPlan + context: _CallContext + + @property + def signature(self) -> type[Signature]: + return self.call_plan.render_signature + + class Adapter: """Base Adapter class. @@ -50,6 +81,7 @@ def __init__( callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = False, native_response_types: list[type[Type]] | None = None, + allow_parallel_tool_calls: bool | None = None, ): """ Args: @@ -60,11 +92,16 @@ def __init__( or `list[dspy.Tool]` types. Defaults to False. native_response_types: List of output field types that should be handled by native LM features rather than adapter parsing. For example, `dspy.Citations` can be populated directly by citation APIs - (e.g., Anthropic's citation feature). Defaults to `[Citations]`. + (e.g., Anthropic's citation feature). Defaults to `[Citations, Reasoning]`. + allow_parallel_tool_calls: Optional provider/tool-call policy. `False` enforces one call per turn for + native and non-native tool calling; `None` preserves provider/model defaults. """ self.callbacks = callbacks or [] self.use_native_function_calling = use_native_function_calling - self.native_response_types = native_response_types or _DEFAULT_NATIVE_RESPONSE_TYPES + self.native_response_types = list( + _DEFAULT_NATIVE_RESPONSE_TYPES if native_response_types is None else native_response_types + ) + self.allow_parallel_tool_calls = allow_parallel_tool_calls def __init_subclass__(cls, **kwargs) -> None: super().__init_subclass__(**kwargs) @@ -73,134 +110,170 @@ def __init_subclass__(cls, **kwargs) -> None: cls.format = with_callbacks(cls.format) cls.parse = with_callbacks(cls.parse) - def _call_preprocess( + def build_call_context(self, lm: BaseLM, lm_kwargs: dict[str, Any] | None = None) -> _CallContext: + return _CallContext( + adapter=self, + use_native_function_calling=self.use_native_function_calling, + allow_parallel_tool_calls=self.allow_parallel_tool_calls, + native_response_types=tuple(self.native_response_types), + lm=_LMCapabilities( + model=lm.model, + model_type=getattr(lm, "model_type", "chat"), + supported_params=frozenset(getattr(lm, "supported_params", set())), + supports_function_calling=bool(getattr(lm, "supports_function_calling", False)), + supports_response_schema=bool(getattr(lm, "supports_response_schema", False)), + supports_reasoning=bool(getattr(lm, "supports_reasoning", False)), + ), + lm_kwargs=dict(lm_kwargs or {}), + lm_default_kwargs=dict(getattr(lm, "kwargs", {}) or {}), + ) + + def _default_call_context(self, *, use_native_tool_calls: bool = False) -> _CallContext: + return _CallContext( + adapter=self, + use_native_function_calling=use_native_tool_calls, + allow_parallel_tool_calls=self.allow_parallel_tool_calls, + native_response_types=tuple(self.native_response_types), + lm=_LMCapabilities( + model="", + model_type="chat", + supported_params=frozenset(), + supports_function_calling=use_native_tool_calls, + supports_response_schema=False, + supports_reasoning=False, + ), + ) + + def value_to_lm_parts(self, value: object, field_info: FieldInfo) -> list[LMPart]: + rendered = format_field_value(field_info=field_info, value=value) + return self.coerce_field_payload_to_lm_parts(rendered) + + def coerce_field_payload_to_lm_parts(self, rendered: object) -> list[LMPart]: + if isinstance(rendered, str): + if "<>" in rendered: + return _split_legacy_custom_type_text_to_parts(rendered) + return [LMTextPart(text=rendered)] + if isinstance(rendered, list): + return LMMessage.model_validate({"role": "user", "content": rendered}).parts + if isinstance(rendered, dict): + if "type" in rendered: + return LMMessage.model_validate({"role": "user", "content": [rendered]}).parts + return [LMTextPart(text=json.dumps(rendered, ensure_ascii=False))] + return [LMTextPart(text=str(rendered))] + + def lm_parts_to_text(self, parts: list[LMPart]) -> str: + texts: list[str] = [] + for part in parts: + text = getattr(part, "text", None) + if text is not None: + texts.append(str(text)) + else: + texts.append(str(part)) + return "".join(texts) + + def lm_parts_to_value(self, parts: list[LMPart], field_info: FieldInfo) -> object: + text = self.lm_parts_to_text(parts) + return parse_value(text, field_info.annotation) + + def wrap_input_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + raise NotImplementedError + + def wrap_output_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + raise NotImplementedError + + def parse_output_fields_to_parts( self, - lm: BaseLM, - lm_kwargs: dict[str, Any], + output: LMOutput, signature: type[Signature], - inputs: dict[str, Any], - ) -> type[Signature]: - # TODO(adapters-plan): This remains the pre-normalized planning hook. It - # mutates `lm_kwargs` and returns only the render signature, which loses - # information we will need for plan-driven rendering/parsing. The next - # stacked PR should replace this with an explicit `_AdapterPlan` that - # records deleted fields, native tools, native output fields, inserted - # messages/parts, and LM config patches. - if self.use_native_function_calling: - tool_call_input_field_name = self._get_tool_call_input_field_name(signature) - tool_call_output_field_name = self._get_tool_call_output_field_name(signature) - - if tool_call_output_field_name and tool_call_input_field_name is None: - raise ValueError( - f"You provided an output field {tool_call_output_field_name} to receive the tool calls information, " - "but did not provide any tools as the input. Please provide a list of tools as the input by adding an " - "input field with type `list[dspy.Tool]`." - ) - - if tool_call_output_field_name and lm.supports_function_calling: - tools = inputs[tool_call_input_field_name] - tools = tools if isinstance(tools, list) else [tools] - - lm_tools = [tool.format_as_litellm_function_call() for tool in tools] - - lm_kwargs["tools"] = lm_tools - - signature_for_native_function_calling = signature.delete(tool_call_output_field_name) - signature_for_native_function_calling = signature_for_native_function_calling.delete( - tool_call_input_field_name - ) - - return signature_for_native_function_calling - - # TODO(adapters-plan): Built-in/native response planning should move out - # of `Type.adapt_to_native_lm_feature()` and into adapter-owned planning - # renderers. Keep this compatibility hook for this boundary-only PR. - # Handle custom types that use native LM features, e.g., reasoning, citations, etc. - for name, field in signature.output_fields.items(): - if ( - isinstance(field.annotation, type) - and field.annotation in self.native_response_types - and issubclass(field.annotation, Type) - ): - signature = field.annotation.adapt_to_native_lm_feature(signature, name, lm, lm_kwargs) + ) -> dict[str, list[LMPart]]: + parsed = self.parse(signature, output.text or "") + return { + field_name: self.value_to_lm_parts(parsed[field_name], field_info) + for field_name, field_info in signature.output_fields.items() + if field_name in parsed + } + + def _parts_to_message_content(self, parts: list[LMPart]) -> str | list[dict[str, Any]]: + return parts_to_openai_content(parts) + + def _make_type_render_context( + self, + *, + field_name: str, + field_info: FieldInfo, + signature: type[Signature], + values: dict[str, Any], + role: str, + context: _CallContext | None, + ) -> _TypeRenderContext: + return _TypeRenderContext( + field_name=field_name, + field_info=field_info, + signature=signature, + values=values, + adapter=self, + call_context=context or self._current_call_context(), + role=role, + ) - return signature + def _current_call_context(self) -> _CallContext: + return getattr(self, "_call_context", None) or self._default_call_context() - def _call_postprocess( + def _default_render_type_value( self, - processed_signature: type[Signature], - original_signature: type[Signature], - outputs: list[dict[str, Any] | str], - lm: BaseLM, - lm_kwargs: dict[str, Any], - ) -> list[dict[str, Any]]: - # TODO(adapters-plan): This still parses legacy adapter output objects. - # PR1 normalizes the LM boundary, then immediately converts back to this - # shape to avoid changing parser semantics. A later PR should parse - # `LMResponse` directly and merge text-parsed fields with explicit - # native fields from `_AdapterPlan`. - values = [] - - tool_call_output_field_name = self._get_tool_call_output_field_name(original_signature) - - for output in outputs: - output_logprobs = None - tool_calls = None - text = output - - if isinstance(output, dict): - text = output["text"] - output_logprobs = output.get("logprobs") - tool_calls = output.get("tool_calls") - - if text: - value = self.parse(processed_signature, text) - for field_name in original_signature.output_fields.keys(): - if field_name not in value: - # We need to set the field not present in the processed signature to None for consistency. - value[field_name] = None - elif tool_calls and tool_call_output_field_name: - value = {} - for field_name in original_signature.output_fields.keys(): - value[field_name] = None - else: - raise AdapterParseError( - adapter_name=type(self).__name__, - signature=original_signature, - lm_response=str(output), - message="The LM returned an empty or null response.", - ) - - if tool_calls and tool_call_output_field_name: - tool_calls = [ - { - "name": v["function"]["name"], - "args": json_repair.loads(v["function"]["arguments"]), - } - for v in tool_calls - ] - value[tool_call_output_field_name] = ToolCalls.from_dict_list(tool_calls) - - # TODO(adapter-types): Once `Type.parse_lm_output(context, output)` is - # the real normalized hook, this should not call the legacy - # provider-shaped `parse_lm_response()` directly. - # Parse custom types that does not rely on the `Adapter.parse()` method - for name, field in original_signature.output_fields.items(): - if ( - isinstance(field.annotation, type) - and field.annotation in self.native_response_types - and issubclass(field.annotation, Type) - ): - parsed_value = field.annotation.parse_lm_response(output) - if parsed_value is not None: - value[name] = parsed_value - - if output_logprobs: - value["logprobs"] = output_logprobs - - values.append(value) - - return values + *, + field_name: str, + field_info: FieldInfo, + signature: type[Signature], + values: dict[str, Any], + value: Any, + role: str, + context: _CallContext | None = None, + ) -> list[LMPart] | None: + annotation = self._type_annotation_class(field_info.annotation) + if annotation is not None and not isinstance(value, annotation): + try: + value = annotation.model_validate(value) + except Exception: + pass + + if not isinstance(value, Type): + return None + + render_ctx = self._make_type_render_context( + field_name=field_name, + field_info=field_info, + signature=signature, + values=values, + role=role, + context=context, + ) + if role == "input": + return value.default_render_input(render_ctx) + return value.default_render_output(render_ctx) + + def _type_annotation_class(self, annotation: Any) -> type[Type] | None: + try: + if isinstance(annotation, type) and issubclass(annotation, Type): + return annotation + except TypeError: + return None + + for arg in get_args(annotation): + annotation_class = self._type_annotation_class(arg) + if annotation_class is not None: + return annotation_class + return None + + def _merge_config_kwargs(self, kwargs: dict[str, Any], config: LMConfig | None) -> dict[str, Any]: + if config is None: + return kwargs + base_config = LMConfig.from_kwargs(**kwargs) + merged_config = _merge_lm_config(base_config, config) + return merged_config.model_dump(exclude_none=True) + + def _merge_lm_config(self, left: LMConfig | None, right: LMConfig | None) -> LMConfig | None: + return _merge_lm_config(left, right) def _render_request( self, From a2086edea4f718945f388a4f3681e325041c5c93 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:28:41 -0400 Subject: [PATCH 15/23] refactor(adapter): route requests through call plans --- dspy/adapters/base.py | 241 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 213 insertions(+), 28 deletions(-) diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index f38364de64..2474dd229c 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -279,19 +279,101 @@ def _render_request( self, lm: BaseLM, lm_kwargs: dict[str, Any], - messages: list[LMMessage | dict[str, Any]], - ) -> LMRequest: - """Build the normalized LM request for the current adapter call path. - - TODO(adapters-plan): This currently receives already-rendered messages. - Once planning lands, this should render from `_AdapterPlan` and apply - planned message/part insertions before creating `LMRequest`. - """ - return LMRequest.from_call( + signature: type[Signature], + demos: list[dict[str, Any]], + inputs: dict[str, Any], + ) -> _RenderedAdapterRequest: + """Render the full signature into a normalized LM request.""" + context = self.build_call_context(lm, lm_kwargs) + call_plan = _AdapterCallPlan.from_signature(signature, inputs, lm_kwargs) + self._prepare_call_plan(call_plan, context) + + request_kwargs = self._prepare_request_kwargs(lm, call_plan.lm_kwargs, signature, call_plan.render_signature, context) + tools = list(call_plan.tools) + if "tools" in request_kwargs: + tools.extend(request_kwargs.pop("tools") or []) + request_kwargs = self._merge_config_kwargs(request_kwargs, call_plan.config) + if not tools: + request_kwargs.pop("tool_choice", None) + + messages = self._format_request_messages(call_plan, demos, context) + request = LMRequest.from_call( model=lm.model, messages=self._coerce_lm_messages(messages), - **lm_kwargs, + tools=tools, + **request_kwargs, ) + return _RenderedAdapterRequest(request=request, call_plan=call_plan, context=context) + + def _type_feature_handlers(self) -> list[_TypeFeatureHandler]: + return [_ToolTypeHandler(), _ReasoningTypeHandler(), _CitationsTypeHandler()] + + def _prepare_call_plan(self, call: _AdapterCallPlan, context: _CallContext) -> None: + for field_name, field_info in call.source_signature.output_fields.items(): + annotation = self._type_annotation_class(field_info.annotation) + if annotation is not None: + call.output_parsers[field_name] = annotation.default_parse_output + + for handler in self._type_feature_handlers(): + handler.prepare(call, context) + + if ( + call.render_signature is not call.source_signature + and call.source_signature.instructions == self._default_signature_instructions(call.source_signature) + ): + call.render_signature = call.render_signature.with_instructions( + self._default_signature_instructions(call.render_signature) + ) + + @staticmethod + def _default_signature_instructions(signature: type[Signature]) -> str: + inputs = ", ".join([f"`{field}`" for field in signature.input_fields]) + outputs = ", ".join([f"`{field}`" for field in signature.output_fields]) + if not outputs: + return f"Given the fields {inputs}, follow the instructions." + return f"Given the fields {inputs}, produce the fields {outputs}." + + def _format_request_messages( + self, + call_plan: _AdapterCallPlan, + demos: list[dict[str, Any]], + context: _CallContext, + ) -> list[LMMessage | dict[str, Any]]: + format_signature = inspect.signature(self.format) + accepts_kwargs = any(param.kind == inspect.Parameter.VAR_KEYWORD for param in format_signature.parameters.values()) + kwargs: dict[str, Any] = {} + if accepts_kwargs or "use_native_tool_calls" in format_signature.parameters: + kwargs["use_native_tool_calls"] = bool(call_plan.tools) + if accepts_kwargs or "context" in format_signature.parameters: + kwargs["context"] = context + if accepts_kwargs or "source_signature" in format_signature.parameters: + kwargs["source_signature"] = call_plan.source_signature + if accepts_kwargs or "prepared_messages" in format_signature.parameters: + kwargs["prepared_messages"] = call_plan.messages + sentinel = object() + previous_context = getattr(self, "_call_context", sentinel) + self._call_context = context + try: + messages = self.format(call_plan.render_signature, demos, call_plan.inputs, **kwargs) + finally: + if previous_context is sentinel: + del self._call_context + else: + self._call_context = previous_context + + if not (accepts_kwargs or "prepared_messages" in format_signature.parameters): + messages = [*messages, *call_plan.messages] + return messages + + def _prepare_request_kwargs( + self, + lm: BaseLM, + lm_kwargs: dict[str, Any], + source_signature: type[Signature], + render_signature: type[Signature], + context: _CallContext, + ) -> dict[str, Any]: + return dict(lm_kwargs) def _call_lm(self, lm: BaseLM, request: LMRequest) -> LMResponse: """Call current `BaseLM` through the normalized request/response boundary. @@ -381,6 +463,116 @@ def _normalize_legacy_outputs(self, outputs: list[dict[str, Any] | str | None], """ return lm_response_from_legacy_outputs(outputs, request) + def _parse_response( + self, + source_signature: type[Signature], + call_plan: _AdapterCallPlan, + response: LMResponse, + context: _CallContext, + ) -> list[dict[str, Any]]: + """Parse a normalized LM response into dictionaries for the source signature.""" + values = [] + + for output in response.outputs: + if output.metadata.get("empty_legacy_outputs"): + continue + + field_parts: dict[str, list[LMPart]] = {} + has_text_output = bool(output.text and call_plan.render_signature.output_fields) + if has_text_output: + field_parts = self.parse_output_fields_to_parts(output, call_plan.render_signature) + + value: dict[str, Any] = {} + for field_name, field_info in call_plan.render_signature.output_fields.items(): + if field_name in field_parts: + value[field_name] = self._parse_field_from_parts( + field_name, + field_info, + call_plan.render_signature, + field_parts[field_name], + output, + context, + call_plan.output_parsers.get(field_name), + ) + + before_type_handler_count = len(value) + for handler in self._type_feature_handlers(): + handler.parse(value, output, call_plan, context) + parsed_by_type_handler = len(value) > before_type_handler_count + + for field_name in source_signature.output_fields: + value.setdefault(field_name, None) + + if not (field_parts or parsed_by_type_handler): + raise AdapterParseError( + adapter_name=type(self).__name__, + signature=source_signature, + lm_response=str(output.to_output_dict()), + message="The LM returned an empty or null response.", + ) + + if output.logprobs is not None: + value["logprobs"] = output.logprobs + + self._validate_tool_call_parallel_policy(source_signature, value) + values.append(value) + + return values + + def _parse_field_from_parts( + self, + field_name: str, + field_info: FieldInfo, + signature: type[Signature], + parts: list[LMPart], + output: LMOutput, + context: _CallContext, + output_parser: _OutputParser | None = None, + ) -> Any: + if output_parser is not None: + return output_parser( + _TypeParseContext( + field_name=field_name, + field_info=field_info, + signature=signature, + adapter=self, + call_context=context, + parts=parts, + lm_output=output, + ) + ) + + annotation = self._type_annotation_class(field_info.annotation) + if annotation is not None: + return annotation.default_parse_output( + _TypeParseContext( + field_name=field_name, + field_info=field_info, + signature=signature, + adapter=self, + call_context=context, + parts=parts, + lm_output=output, + ) + ) + return self.lm_parts_to_value(parts, field_info) + + def _validate_tool_call_parallel_policy(self, signature: type[Signature], value: dict[str, Any]) -> None: + if self.allow_parallel_tool_calls is not False: + return + + tool_call_output_field_name = self._get_tool_call_output_field_name(signature) + if not tool_call_output_field_name or tool_call_output_field_name not in value: + return + + tool_calls = value[tool_call_output_field_name] + if tool_calls is None: + return + if not isinstance(tool_calls, ToolCalls): + tool_calls = ToolCalls.model_validate(tool_calls) + value[tool_call_output_field_name] = tool_calls + tool_calls.validate_max_items(1) + def __call__( self, lm: BaseLM, @@ -405,16 +597,9 @@ def __call__( List of dictionaries representing parsed LM responses. Each dictionary contains keys matching the signature's output field names. For multiple generations (n > 1), returns multiple dictionaries. """ - processed_signature = self._call_preprocess(lm, lm_kwargs, signature, inputs) - messages = self.format(processed_signature, demos, inputs) - request = self._render_request(lm, lm_kwargs, messages) - response = self._call_lm(lm, request) - # TODO(adapters-response): We normalize at the LM boundary, but still - # convert back to legacy postprocess dictionaries here to keep this PR - # behavior-preserving. Replace with direct `LMResponse` parsing once the - # explicit adapter plan exists. - outputs = legacy_outputs_from_lm_response(response) - return self._call_postprocess(processed_signature, signature, outputs, lm, lm_kwargs) + rendered = self._render_request(lm, lm_kwargs, signature, demos, inputs) + response = self._call_lm(lm, rendered.request) + return self._parse_response(signature, rendered.call_plan, response, rendered.context) async def acall( self, @@ -424,20 +609,20 @@ async def acall( demos: list[dict[str, Any]], inputs: dict[str, Any], ) -> list[dict[str, Any]]: - processed_signature = self._call_preprocess(lm, lm_kwargs, signature, inputs) - messages = self.format(processed_signature, demos, inputs) - request = self._render_request(lm, lm_kwargs, messages) - response = await self._acall_lm(lm, request) - # TODO(adapters-response): Keep in sync with `__call__()` until both use - # direct `LMResponse` parsing. - outputs = legacy_outputs_from_lm_response(response) - return self._call_postprocess(processed_signature, signature, outputs, lm, lm_kwargs) + rendered = self._render_request(lm, lm_kwargs, signature, demos, inputs) + response = await self._acall_lm(lm, rendered.request) + return self._parse_response(signature, rendered.call_plan, response, rendered.context) def format( self, signature: type[Signature], demos: list[dict[str, Any]], inputs: dict[str, Any], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + source_signature: type[Signature] | None = None, + prepared_messages: list[LMMessage] | None = None, ) -> list[dict[str, Any]]: """Format the input messages for the LM call. From 45757606593925f19d4956c3e794be500b68c229 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:30:25 -0400 Subject: [PATCH 16/23] refactor(adapter): render normalized request messages --- dspy/adapters/base.py | 102 ++++++++++++++++++++++------ tests/adapters/test_chat_adapter.py | 2 +- 2 files changed, 82 insertions(+), 22 deletions(-) diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index 2474dd229c..e275cdf098 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -663,33 +663,93 @@ def format( Returns: A list of multiturn messages as expected by the LM. """ + return [ + message_to_openai_chat(message) + for message in self.render_messages( + signature, + demos, + inputs, + use_native_tool_calls=use_native_tool_calls, + context=context, + source_signature=source_signature, + prepared_messages=prepared_messages, + ) + ] + + def render_messages( + self, + signature: type[Signature], + demos: list[dict[str, Any]], + inputs: dict[str, Any], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + source_signature: type[Signature] | None = None, + prepared_messages: list[LMMessage] | None = None, + ) -> list[LMMessage]: + """Render adapter messages as normalized LM messages.""" + context = context or self._default_call_context(use_native_tool_calls=use_native_tool_calls) + source_signature = source_signature or signature inputs_copy = dict(inputs) - # If the signature and inputs have conversation history, we need to format the conversation history and - # remove the history field from the signature. - history_field_name = self._get_history_field_name(signature) + render_signature = signature + conversation_history: list[LMMessage] = [] + history_field_name = self._get_history_field_name(source_signature) if history_field_name: - # In order to format the conversation history, we need to remove the history field from the signature. - signature_without_history = signature.delete(history_field_name) - conversation_history = self.format_conversation_history( - signature_without_history, - history_field_name, + if history_field_name in render_signature.fields: + previous_render_signature = render_signature + render_signature = render_signature.delete(history_field_name) + if previous_render_signature.instructions == self._default_signature_instructions( + previous_render_signature + ): + render_signature = render_signature.with_instructions( + self._default_signature_instructions(render_signature) + ) + history_obj = inputs_copy.pop(history_field_name, None) + if history_obj is not None: + conversation_history = self.format_history( + history_obj, + source_signature.delete(history_field_name), + context=context, + ) + + messages: list[LMMessage | dict[str, Any]] = [ + {"role": "system", "content": self.format_system_message(render_signature)} + ] + messages.extend( + self.format_demos( + source_signature.delete(history_field_name) if history_field_name else source_signature, + demos, + context=context, + ) + ) + messages.extend(conversation_history) + messages.extend(prepared_messages or []) + messages.extend( + self._format_input_messages( + render_signature, inputs_copy, + main_request=True, + context=context, ) + ) - messages = [] - system_message = self.format_system_message(signature) - messages.append({"role": "system", "content": system_message}) - messages.extend(self.format_demos(signature, demos)) - if history_field_name: - # Conversation history and current input - content = self.format_user_message_content(signature_without_history, inputs_copy, main_request=True) - messages.extend(conversation_history) - messages.append({"role": "user", "content": content}) - else: - # Only current input - content = self.format_user_message_content(signature, inputs_copy, main_request=True) - messages.append({"role": "user", "content": content}) + return self._coerce_lm_messages(messages) + + def format_history( + self, + history: History, + signature: type[Signature], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + ) -> list[LMMessage]: + """Render stored field history using this adapter's field format.""" + context = context or self._default_call_context(use_native_tool_calls=use_native_tool_calls) + messages: list[LMMessage] = [] + for message_idx, entry in enumerate(history.messages): + if not isinstance(entry, dict): + continue return [_expand_legacy_custom_type_markers_in_chat_message(message) for message in messages] diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py index 17c1603766..ef2cc38a5e 100644 --- a/tests/adapters/test_chat_adapter.py +++ b/tests/adapters/test_chat_adapter.py @@ -368,7 +368,7 @@ class HistorySignature(dspy.Signature): "\n" "[[ ## completed ## ]]\n" "In adhering to this structure, your objective is: \n" - " Given the fields `history`, `question`, produce the fields `answer`."}, + " Given the fields `question`, produce the fields `answer`."}, {"role": "user", "content": "[[ ## question ## ]]\nWhat is 1+1?"}, {"role": "assistant", "content": "[[ ## answer ## ]]\n2\n\n[[ ## completed ## ]]\n"}, {"role": "user", "content": "[[ ## question ## ]]\nWhat is 2+2?"}, From 556f04c62787963dc576747e8643042b75c7da85 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:30:46 -0400 Subject: [PATCH 17/23] refactor(adapter): replay history through call plans --- dspy/adapters/base.py | 321 ++++++++++++++++++++- tests/adapters/test_citation.py | 62 +++- tests/adapters/test_history_lm_messages.py | 95 ++++++ tests/adapters/test_json_adapter.py | 85 +++--- tests/adapters/test_reasoning.py | 33 +++ tests/adapters/test_xml_adapter.py | 19 +- 6 files changed, 548 insertions(+), 67 deletions(-) create mode 100644 tests/adapters/test_history_lm_messages.py diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index e275cdf098..a98cc3ff57 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -751,7 +751,326 @@ def format_history( if not isinstance(entry, dict): continue - return [_expand_legacy_custom_type_markers_in_chat_message(message) for message in messages] + input_values = {key: value for key, value in entry.items() if key in signature.input_fields} + tool_result_values = { + key: value + for key, value in input_values.items() + if self._coerce_tool_call_results_value(value, signature.input_fields.get(key)) is not None + } + output_values = {key: value for key, value in entry.items() if key in signature.output_fields} + unknown_values = { + key: value for key, value in entry.items() if key not in input_values and key not in output_values + } + + turn_call = _AdapterCallPlan.from_signature(signature, input_values, context.lm_kwargs) + self._prepare_call_plan(turn_call, context) + + regular_input_values = { + key: turn_call.inputs[key] + for key in turn_call.render_signature.input_fields + if key in turn_call.inputs and key not in tool_result_values + } + if regular_input_values: + messages.extend( + self._format_input_messages( + turn_call.render_signature, + regular_input_values, + main_request=False, + context=context, + ) + ) + + assistant_values = dict(output_values) + if not turn_call.messages: + assistant_values.update(unknown_values) + + if assistant_values: + messages.extend( + self._format_output_messages_for_call( + turn_call, + assistant_values, + message_idx, + context=context, + ) + ) + + messages.extend(turn_call.messages) + + fallback_tool_result_values = { + key: value + for key, value in tool_result_values.items() + if key in turn_call.render_signature.input_fields and not turn_call.messages + } + if fallback_tool_result_values: + messages.extend( + self._format_input_messages( + turn_call.render_signature, + fallback_tool_result_values, + main_request=False, + context=context, + ) + ) + + if (turn_call.messages or fallback_tool_result_values) and unknown_values: + messages.extend( + self._format_output_messages_for_call( + turn_call, + unknown_values, + message_idx, + context=context, + ) + ) + + return messages + + def _format_input_messages( + self, + signature: type[Signature], + inputs: dict[str, Any], + *, + main_request: bool, + context: _CallContext | None = None, + prefix: str = "", + suffix: str = "", + ) -> list[LMMessage]: + context = context or self._current_call_context() + regular_inputs = self._drop_absent_optional_inputs(signature, dict(inputs)) + if self._should_render_input_as_parts(signature, regular_inputs, context): + parts: list[LMPart] = [] + if prefix: + parts.append(LMTextPart(text=prefix)) + field_seen = False + for key, field_info in signature.input_fields.items(): + if key not in regular_inputs: + continue + if field_seen: + parts.append(LMTextPart(text="\n\n")) + value_parts = self._default_render_type_value( + field_name=key, + field_info=field_info, + signature=signature, + values=regular_inputs, + value=regular_inputs[key], + role="input", + context=context, + ) + if value_parts is None: + value_parts = self.value_to_lm_parts(regular_inputs[key], field_info) + parts.extend(self.wrap_input_field_parts(key, value_parts)) + field_seen = True + if main_request: + output_requirements_fn = getattr(self, "user_message_output_requirements", lambda _signature: None) + output_requirements = output_requirements_fn(signature) + if output_requirements is not None: + if parts: + parts.append(LMTextPart(text="\n\n")) + parts.append(LMTextPart(text=output_requirements)) + if suffix: + parts.append(LMTextPart(text=suffix)) + parts = self._merge_adjacent_text_parts(parts) + return [LMMessage(role="user", parts=parts)] if parts else [] + + content = self.format_user_message_content( + signature, + regular_inputs, + prefix=prefix, + suffix=suffix, + main_request=main_request, + ) + if self._has_content(content): + return [self._content_message("user", content)] + return [] + + def _format_output_messages_for_call( + self, + call: _AdapterCallPlan, + outputs: dict[str, Any], + message_idx: int, + *, + context: _CallContext | None = None, + missing_field_message: str | None = "Not supplied for this conversation history message. ", + ) -> list[LMMessage]: + context = context or self._current_call_context() + regular_outputs = {} + native_parts: list[LMPart] = [] + native_messages: list[LMMessage] = [] + + for key, value in outputs.items(): + if key in call.render_signature.output_fields: + regular_outputs[key] = value + continue + + rendered_type = self._format_type_feature_output(key, value, call, context) + if rendered_type is not None and rendered_type.consumed: + native_parts.extend(rendered_type.parts) + native_messages.extend(rendered_type.messages) + for message in rendered_type.messages: + if message.role == "assistant": + native_parts.extend(message.parts) + continue + regular_outputs[key] = value + + if native_parts: + content = self._format_native_output_content( + call.render_signature, + regular_outputs, + missing_field_message=missing_field_message, + ) + parts: list[LMPart] = [] + if content: + parts.append(LMTextPart(text=content)) + parts.extend(native_parts) + return [LMMessage(role="assistant", parts=parts)] + + if native_messages: + return native_messages + + return [ + self._content_message( + "assistant", + self._format_output_content( + call.render_signature, + regular_outputs, + missing_field_message=missing_field_message, + ), + ) + ] + + def _format_type_feature_output( + self, + field_name: str, + value: object, + call: _AdapterCallPlan, + context: _CallContext, + ) -> _RenderedTypeOutput | None: + for handler in self._type_feature_handlers(): + formatter = getattr(handler, "format_output", None) + if formatter is None: + continue + rendered_type = formatter(field_name, value, call, context) + if rendered_type is not None: + return rendered_type + return None + + def _format_output_content( + self, + signature: type[Signature], + outputs: dict[str, Any], + *, + missing_field_message: str | None, + ) -> str: + signature_outputs = {key: value for key, value in outputs.items() if key in signature.output_fields} + unknown_outputs = {key: value for key, value in outputs.items() if key not in signature.output_fields} + if signature_outputs and not unknown_outputs: + return self.format_assistant_message_content( + signature, + signature_outputs, + missing_field_message=missing_field_message, + ) + + sections = [] + if signature_outputs: + sections.append( + self.format_assistant_message_content( + signature, + signature_outputs, + missing_field_message=missing_field_message, + ).strip() + ) + + for key, value in unknown_outputs.items(): + sections.append(f"[[ ## {key} ## ]]\n{self._format_value(value)}") + + sections.append("[[ ## completed ## ]]") + return "\n\n".join(section for section in sections if section) + + def _format_native_output_content( + self, + signature: type[Signature], + outputs: dict[str, Any], + *, + missing_field_message: str | None, + ) -> str | None: + if not outputs: + return None + if len(outputs) == 1: + return str(next(iter(outputs.values()))) + return self._format_output_content(signature, outputs, missing_field_message=missing_field_message) + + @staticmethod + def _drop_absent_optional_inputs(signature: type[Signature], inputs: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in inputs.items() + if not (value is None and key in signature.input_fields and signature.input_fields[key].default is None) + } + + def _should_render_input_as_parts( + self, + signature: type[Signature], + inputs: dict[str, Any], + context: _CallContext, + ) -> bool: + for key, value in inputs.items(): + field_info = signature.input_fields.get(key) + if field_info is None: + continue + parts = self._default_render_type_value( + field_name=key, + field_info=field_info, + signature=signature, + values=inputs, + value=value, + role="input", + context=context, + ) + if parts is not None and any(not isinstance(part, LMTextPart) for part in parts): + return True + return False + + @staticmethod + def _merge_adjacent_text_parts(parts: list[LMPart]) -> list[LMPart]: + merged: list[LMPart] = [] + for part in parts: + if ( + isinstance(part, LMTextPart) + and merged + and isinstance(merged[-1], LMTextPart) + and not part.metadata + and not merged[-1].metadata + ): + merged[-1] = LMTextPart(text=merged[-1].text + part.text) + else: + merged.append(part) + return merged + + @staticmethod + def _coerce_tool_call_results_value(field_value: Any, field_info: Any = None) -> ToolCallResults | None: + if field_value is None: + return None + if isinstance(field_value, ToolCallResults): + return field_value + + annotation = getattr(field_info, "annotation", None) + if annotation is ToolCallResults or ToolCallResults in get_args(annotation): + return ToolCallResults.model_validate(field_value) + return None + + @staticmethod + def _format_value(value: Any) -> str: + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return str(value) + + @staticmethod + def _has_content(content: Any) -> bool: + if isinstance(content, str): + return bool(content.strip()) + return bool(content) + + @staticmethod + def _content_message(role: str, content: Any) -> LMMessage: + return LMMessage.model_validate({"role": role, "content": content}) def format_system_message(self, signature: type[Signature]) -> str: """Format the system message for the LM call. diff --git a/tests/adapters/test_citation.py b/tests/adapters/test_citation.py index 3c2a7f75e6..a7bed4372e 100644 --- a/tests/adapters/test_citation.py +++ b/tests/adapters/test_citation.py @@ -2,6 +2,9 @@ import pytest import dspy +from dspy.adapters._type_feature_handlers import _CitationsTypeHandler +from dspy.adapters._type_runtime import _AdapterCallPlan +from dspy.core.types import LMCitationPart, LMOutput from dspy.experimental import Citations @@ -106,6 +109,52 @@ def test_citations_format(): assert formatted[1]["document_title"] == "Source" +def test_citations_type_handler_uses_native_lm_output_and_parts(): + class CitationSignature(dspy.Signature): + question: str = dspy.InputField() + citations: Citations = dspy.OutputField() + + adapter = dspy.Adapter(native_response_types=[Citations]) + lm = dspy.LM(model="anthropic/claude-3-5-sonnet-20241022") + context = adapter.build_call_context(lm) + call = _AdapterCallPlan.from_signature(CitationSignature, {"question": "Q?"}, {}) + type_handler = _CitationsTypeHandler() + type_handler.prepare(call, context) + + citation = Citations.Citation( + cited_text="The sky is blue", + document_index=0, + document_title="Weather Guide", + start_char_index=10, + end_char_index=25, + supported_text="The sky is blue", + ) + rendered = type_handler.format_output("citations", Citations(citations=[citation]), call, context) + + assert "citations" not in call.render_signature.output_fields + assert rendered.parts[0].text == "The sky is blue" + + values = {} + type_handler.parse( + values, + LMOutput( + parts=[ + LMCitationPart( + text="The sky is blue", + title="Weather Guide", + metadata={"document_index": 0, "start_char_index": 10, "end_char_index": 25}, + ) + ] + ), + call, + context, + ) + + parsed = values["citations"] + assert parsed.citations[0].cited_text == "The sky is blue" + assert parsed.citations[0].document_title == "Weather Guide" + + def test_citations_from_dict_list(): citations_data = [ { @@ -151,13 +200,12 @@ class CitationSignature(Signature): ] }] - result = adapter._call_postprocess( - CitationSignature.delete("citations"), - CitationSignature, - outputs, - dspy.LM(model="anthropic/claude-3-5-sonnet-20241022"), - lm_kwargs={}, - ) + from dspy.clients.openai_format import lm_response_from_legacy_outputs + + lm = dspy.LM(model="anthropic/claude-3-5-sonnet-20241022") + rendered = adapter._render_request(lm, {}, CitationSignature, [], {"question": "What color is the sky?"}) + response = lm_response_from_legacy_outputs(outputs, rendered.request) + result = adapter._parse_response(CitationSignature, rendered.call_plan, response, rendered.context) assert len(result) == 1 assert "citations" in result[0] diff --git a/tests/adapters/test_history_lm_messages.py b/tests/adapters/test_history_lm_messages.py new file mode 100644 index 0000000000..66867dd6ab --- /dev/null +++ b/tests/adapters/test_history_lm_messages.py @@ -0,0 +1,95 @@ +import json + +import dspy +from dspy.adapters.types.history import History +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls + + +def add(a: int, b: int) -> int: + return a + b + + +class NativeToolLM: + model = "openai/gpt-5-nano" + supported_params = frozenset() + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + + def __call__(self, messages, **kwargs): + self.messages = messages + self.kwargs = kwargs + return [{"text": "[[ ## next_thought ## ]]\nDone.\n\n[[ ## completed ## ]]"}] + + +def test_history_renders_native_tool_messages_through_lm_message_path(): + adapter = dspy.ChatAdapter(use_native_function_calling=True) + signature = ( + dspy.Signature({}, "Do the task.") + .append("question", dspy.InputField(), type_=str) + .append("history", dspy.InputField(), type_=dspy.History) + .append("tools", dspy.InputField(), type_=list[dspy.Tool]) + .append("previous_tool_results", dspy.InputField(), type_=dspy.ToolCallResults) + .append("next_thought", dspy.OutputField(), type_=str) + .append("tool_calls", dspy.OutputField(), type_=dspy.ToolCalls) + ) + history = History(messages=[]) + history.append( + { + "question": "What is 1+2?", + "next_thought": "I should add the two numbers.", + "tool_calls": ToolCalls.from_dict_list( + [{"name": "add", "args": {"a": 1, "b": 2}, "id": "call_add"}] + ), + "previous_tool_results": ToolCallResults.from_dict_list( + [{"call_id": "call_add", "name": "add", "value": 3}] + ), + } + ) + + lm = NativeToolLM() + adapter(lm, {}, signature, [], {"history": history, "tools": [Tool(add)]}) + + assistant_message = next(message for message in lm.messages if message["role"] == "assistant") + tool_message = next(message for message in lm.messages if message["role"] == "tool") + final_user_message = lm.messages[-1] + + assert assistant_message["content"] == "I should add the two numbers." + assert assistant_message["tool_calls"][0]["id"] == "call_add" + assert assistant_message["tool_calls"][0]["function"]["name"] == "add" + assert json.loads(assistant_message["tool_calls"][0]["function"]["arguments"]) == {"a": 1, "b": 2} + assert tool_message == {"role": "tool", "content": "3", "tool_call_id": "call_add", "name": "add"} + assert "What is 1+2?" not in final_user_message["content"] + assert "Respond with the corresponding output fields" in final_user_message["content"] + + +def test_current_tool_result_value_renders_native_tool_messages_independent_of_field_name(): + adapter = dspy.ChatAdapter(use_native_function_calling=True) + signature = ( + dspy.Signature({}, "Continue the task.") + .append("question", dspy.InputField(), type_=str) + .append("tools", dspy.InputField(), type_=list[dspy.Tool]) + .append("previous_tool_results", dspy.InputField(), type_=dspy.ToolCallResults) + .append("next_thought", dspy.OutputField(), type_=str) + .append("tool_calls", dspy.OutputField(), type_=dspy.ToolCalls) + ) + tool_results = ToolCallResults.from_dict_list([{"call_id": "call_add", "name": "add", "value": 3}]) + + lm = NativeToolLM() + adapter( + lm, + {}, + signature, + [], + { + "question": "What is 1+2?", + "tools": [Tool(add)], + "previous_tool_results": tool_results, + }, + ) + + tool_message = next(message for message in lm.messages if message["role"] == "tool") + user_messages = [message for message in lm.messages if message["role"] == "user"] + + assert tool_message == {"role": "tool", "content": "3", "tool_call_id": "call_add", "name": "add"} + assert all("previous_tool_results" not in (message["content"] or "") for message in user_messages) diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py index c6dd1f679d..2cd7f2c5a5 100644 --- a/tests/adapters/test_json_adapter.py +++ b/tests/adapters/test_json_adapter.py @@ -231,11 +231,10 @@ class RichRenderingSignature(dspy.Signature): expected_messages = [{"role": "system", "content": 'Your input fields are:\n' - '1. `history` (History): \n' - '2. `image` (Image): \n' - '3. `tools` (list[Tool]): \n' - '4. `profile` (Profile): \n' - '5. `question` (str):\n' + '1. `image` (Image): \n' + '2. `tools` (list[Tool]): \n' + '3. `profile` (Profile): \n' + '4. `question` (str):\n' 'Your output fields are:\n' '1. `answer` (AnswerCard):\n' 'All interactions will be structured in the following way, with the appropriate ' @@ -243,9 +242,6 @@ class RichRenderingSignature(dspy.Signature): '\n' 'Inputs will have the following structure:\n' '\n' - '[[ ## history ## ]]\n' - '{history}\n' - '\n' '[[ ## image ## ]]\n' '{image}\n' '\n' @@ -270,19 +266,14 @@ class RichRenderingSignature(dspy.Signature): 'In adhering to this structure, your objective is: \n' ' Answer using all supplied context.'}, {"role": "user", - "content": [{"type": "text", - "text": "This is an example of the task, though some input or output fields are not " - "supplied.\n" - "\n" - "[[ ## image ## ]]\n"}, + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, {"type": "text", "text": '\n' '\n' '[[ ## tools ## ]]\n' - '["search, whose description is Search for documents.. It ' - "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " - '\'default\': 3}}."]\n' + '["search, whose description is Search for documents.. It takes ' + "arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', 'default': 3}}.\"]\n" '\n' '[[ ## profile ## ]]\n' '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' @@ -316,15 +307,15 @@ class RichRenderingSignature(dspy.Signature): ' }\n' '}'}, {"role": "user", - "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + "content": [{"type": "text", + "text": "[[ ## image ## ]]\n"}, {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, {"type": "text", "text": '\n' '\n' '[[ ## tools ## ]]\n' - '["search, whose description is Search for documents.. It ' - "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " - '\'default\': 3}}."]\n' + '["search, whose description is Search for documents.. It takes ' + "arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', 'default': 3}}.\"]\n" '\n' '[[ ## profile ## ]]\n' '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' @@ -591,12 +582,10 @@ class NativeToolSignature(dspy.Signature): "\n" "{}\n" "In adhering to this structure, your objective is: \n" - " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, + " Given the fields `question`, follow the instructions."}, {"role": "user", "content": "[[ ## question ## ]]\n" - "Q?\n" - "\n" - "Respond with a JSON object in the following order of fields: ."}] + "Q?"}] assert messages == expected_messages expected_lm_kwargs = {"tools": [{"type": "function", "function": {"name": "search", @@ -640,8 +629,10 @@ class ToolCallsSignature(dspy.Signature): ' "tool_calls": "{tool_calls} # note: the value you produce must adhere to ' 'the JSON schema: {\\"type\\": \\"object\\", \\"$defs\\": {\\"ToolCall\\": ' '{\\"type\\": \\"object\\", \\"properties\\": {\\"args\\": {\\"type\\": \\"object\\", ' - '\\"additionalProperties\\": true, \\"title\\": \\"Args\\"}, \\"name\\": {\\"type\\": ' - '\\"string\\", \\"title\\": \\"Name\\"}}, \\"required\\": [\\"name\\", \\"args\\"], ' + '\\"additionalProperties\\": true, \\"title\\": \\"Args\\"}, \\"id\\": {\\"anyOf\\": ' + '[{\\"type\\": \\"string\\"}, {\\"type\\": \\"null\\"}], \\"default\\": null, ' + '\\"title\\": \\"Id\\"}, \\"name\\": {\\"type\\": \\"string\\", \\"title\\": ' + '\\"Name\\"}}, \\"required\\": [\\"name\\", \\"args\\"], ' '\\"title\\": \\"ToolCall\\"}}, \\"properties\\": {\\"tool_calls\\": {\\"type\\": ' '\\"array\\", \\"items\\": {\\"$ref\\": \\"#/$defs/ToolCall\\"}, \\"title\\": \\"Tool ' 'Calls\\"}}, \\"required\\": [\\"tool_calls\\"], \\"title\\": \\"ToolCalls\\"}"\n' @@ -650,20 +641,17 @@ class ToolCallsSignature(dspy.Signature): ' Given the fields `question`, produce the fields `tool_calls`.'}, {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, {"role": "assistant", - "content": '{\n' - ' "tool_calls": {\n' - ' "tool_calls": [\n' - ' {\n' - ' "type": "function",\n' - ' "function": {\n' - ' "name": "search",\n' - ' "arguments": {\n' - ' "query": "cats"\n' - ' }\n' - ' }\n' - ' }\n' - ' ]\n' - ' }\n' + "content": '{\n' + ' "tool_calls": {\n' + ' "tool_calls": [\n' + ' {\n' + ' "name": "search",\n' + ' "args": {\n' + ' "query": "cats"\n' + ' }\n' + ' }\n' + ' ]\n' + ' }\n' '}'}, {"role": "user", "content": "[[ ## question ## ]]\n" @@ -1500,7 +1488,13 @@ def get_weather(city: str) -> str: ) assert result[0]["tool_calls"] == dspy.ToolCalls( - tool_calls=[dspy.ToolCalls.ToolCall(name="get_weather", args={"city": "Paris"})] + tool_calls=[ + dspy.ToolCalls.ToolCall( + name="get_weather", + args={"city": "Paris"}, + id="call_pQm8ajtSMxgA0nrzK2ivFmxG", + ) + ] ) # `answer` is not present, so we set it to None assert result[0]["answer"] is None @@ -1574,13 +1568,14 @@ class MySignature(dspy.Signature): ], model="anthropic/claude-3-7-sonnet-20250219", ) - modified_signature = adapter._call_preprocess( - dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), - {}, + messages, _ = format_messages_and_lm_kwargs( + adapter, MySignature, + [], {"question": "What is the capital of France?"}, + lm=dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), ) - assert "reasoning" not in modified_signature.output_fields + assert '"reasoning"' not in messages[0]["content"] result = adapter( dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), diff --git a/tests/adapters/test_reasoning.py b/tests/adapters/test_reasoning.py index 90812b81cb..80143775b3 100644 --- a/tests/adapters/test_reasoning.py +++ b/tests/adapters/test_reasoning.py @@ -1,6 +1,9 @@ import pytest import dspy +from dspy.adapters._type_feature_handlers import _ReasoningTypeHandler +from dspy.adapters._type_runtime import _AdapterCallPlan +from dspy.core.types import LMOutput, LMThinkingPart def test_reasoning_basic_operations(): @@ -109,3 +112,33 @@ def test_reasoning_error_message(): with pytest.raises(AttributeError, match="`Reasoning` object has no attribute 'nonexistent_method'"): reasoning.nonexistent_method + + +def test_reasoning_type_handler_uses_native_lm_output_and_config(): + class ReasoningSignature(dspy.Signature): + question: str = dspy.InputField() + reasoning: dspy.Reasoning = dspy.OutputField() + + class NativeReasoningLM: + model = "anthropic/claude-3-7-sonnet" + model_type = "chat" + supports_function_calling = False + supports_reasoning = True + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + adapter = dspy.Adapter() + context = adapter.build_call_context(NativeReasoningLM()) + call = _AdapterCallPlan.from_signature(ReasoningSignature, {"question": "Q?"}, {}) + type_handler = _ReasoningTypeHandler() + + type_handler.prepare(call, context) + + assert "reasoning" not in call.render_signature.output_fields + assert call.config.reasoning.effort == "low" + + values = {} + type_handler.parse(values, LMOutput(parts=[LMThinkingPart(text="think")]), call, context) + + assert values["reasoning"] == dspy.Reasoning(content="think") diff --git a/tests/adapters/test_xml_adapter.py b/tests/adapters/test_xml_adapter.py index ee300ba3cd..f227f33261 100644 --- a/tests/adapters/test_xml_adapter.py +++ b/tests/adapters/test_xml_adapter.py @@ -534,20 +534,15 @@ class RichRenderingSignature(dspy.Signature): expected_messages = [{"role": "system", "content": 'Your input fields are:\n' - '1. `history` (History): \n' - '2. `image` (Image): \n' - '3. `tools` (list[Tool]): \n' - '4. `profile` (Profile): \n' - '5. `question` (str):\n' + '1. `image` (Image): \n' + '2. `tools` (list[Tool]): \n' + '3. `profile` (Profile): \n' + '4. `question` (str):\n' 'Your output fields are:\n' '1. `answer` (AnswerCard):\n' 'All interactions will be structured in the following way, with the appropriate ' 'values filled in.\n' '\n' - '\n' - '{history}\n' - '\n' - '\n' '\n' '{image}\n' '\n' @@ -573,11 +568,7 @@ class RichRenderingSignature(dspy.Signature): 'In adhering to this structure, your objective is: \n' ' Answer using all supplied context.'}, {"role": "user", - "content": [{"type": "text", - "text": "This is an example of the task, though some input or output fields are not " - "supplied.\n" - "\n" - "\n"}, + "content": [{"type": "text", "text": "\n"}, {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, {"type": "text", "text": '\n' From 52d0fd87e641fe9403a22c1ab5311cfd1395cb3a Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:31:05 -0400 Subject: [PATCH 18/23] refactor(adapter): render demos through call plans --- dspy/adapters/base.py | 139 ++++++++++++++++++++++++++---------------- 1 file changed, 88 insertions(+), 51 deletions(-) diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index a98cc3ff57..d9a29e772b 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -1170,7 +1170,14 @@ def format_assistant_message_content( """ raise NotImplementedError - def format_demos(self, signature: type[Signature], demos: list[dict[str, Any]]) -> list[dict[str, Any]]: + def format_demos( + self, + signature: type[Signature], + demos: list[dict[str, Any]], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + ) -> list[LMMessage | dict[str, Any]]: """Format the few-shot examples. This method formats the few-shot examples as multiturn messages. @@ -1183,12 +1190,14 @@ def format_demos(self, signature: type[Signature], demos: list[dict[str, Any]]) Returns: A list of multiturn messages. """ + context = context or self._default_call_context(use_native_tool_calls=use_native_tool_calls) + active_fields = signature.fields complete_demos = [] incomplete_demos = [] for demo in demos: # Check if all fields are present and not None - is_complete = all(k in demo and demo[k] is not None for k in signature.fields) + is_complete = all(k in demo and demo[k] is not None for k in active_fields) # Check if demo has at least one input and one output field has_input = any(k in demo for k in signature.input_fields) @@ -1204,56 +1213,99 @@ def format_demos(self, signature: type[Signature], demos: list[dict[str, Any]]) incomplete_demo_prefix = "This is an example of the task, though some input or output fields are not supplied." for demo in incomplete_demos: - messages.append( - { - "role": "user", - "content": self.format_user_message_content(signature, demo, prefix=incomplete_demo_prefix), - } + demo_call = _AdapterCallPlan.from_signature( + signature, + {key: demo[key] for key in signature.input_fields if key in demo}, + context.lm_kwargs, ) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content( - signature, demo, missing_field_message="Not supplied for this particular example. " - ), - } + self._prepare_call_plan(demo_call, context) + demo_inputs = { + key: demo_call.inputs[key] + for key in demo_call.render_signature.input_fields + if key in demo_call.inputs + } + demo_outputs = {key: demo[key] for key in signature.output_fields if key in demo} + messages.extend( + self._format_input_messages( + demo_call.render_signature, + demo_inputs, + main_request=False, + context=context, + prefix=incomplete_demo_prefix, + ) ) + messages.extend( + self._format_output_messages_for_call( + demo_call, + demo_outputs, + len(messages), + context=context, + missing_field_message="Not supplied for this particular example. ", + ) + ) + messages.extend(demo_call.messages) for demo in complete_demos: - messages.append({"role": "user", "content": self.format_user_message_content(signature, demo)}) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content( - signature, demo, missing_field_message="Not supplied for this conversation history message. " - ), - } + demo_call = _AdapterCallPlan.from_signature( + signature, + {key: demo[key] for key in signature.input_fields if key in demo}, + context.lm_kwargs, ) + self._prepare_call_plan(demo_call, context) + demo_inputs = { + key: demo_call.inputs[key] + for key in demo_call.render_signature.input_fields + if key in demo_call.inputs + } + demo_outputs = {key: demo[key] for key in signature.output_fields if key in demo} + messages.extend( + self._format_input_messages( + demo_call.render_signature, + demo_inputs, + main_request=False, + context=context, + ) + ) + messages.extend( + self._format_output_messages_for_call( + demo_call, + demo_outputs, + len(messages), + context=context, + missing_field_message="Not supplied for this conversation history message. ", + ) + ) + messages.extend(demo_call.messages) return messages - def _get_history_field_name(self, signature: type[Signature]) -> bool: + def _get_history_field_name(self, signature: type[Signature]) -> str | None: for name, field in signature.input_fields.items(): if field.annotation == History: return name return None - def _get_tool_call_input_field_name(self, signature: type[Signature]) -> bool: - for name, field in signature.input_fields.items(): - # Look for annotation `list[dspy.Tool]` or `dspy.Tool` - origin = get_origin(field.annotation) - if origin is list and field.annotation.__args__[0] == Tool: - return name - if field.annotation == Tool: - return name - return None + def _get_tool_call_output_field_name(self, signature: type[Signature]) -> str | None: + return self._find_tool_call_output_field_name(signature) - def _get_tool_call_output_field_name(self, signature: type[Signature]) -> bool: + @classmethod + def _find_tool_call_output_field_name(cls, signature: type[Signature]) -> str | None: for name, field in signature.output_fields.items(): - if field.annotation == ToolCalls: + if cls._annotation_includes_static(field.annotation, ToolCalls): return name return None + @classmethod + def _annotation_includes_static(cls, annotation: Any, target: type) -> bool: + if annotation is target: + return True + return any(cls._annotation_includes_static(arg, target) for arg in get_args(annotation)) + + def force_tool_call_config(self, tool_name: str) -> dict[str, Any]: + if not self.use_native_function_calling: + return {} + return {"tool_choice": {"mode": "required", "allowed": [tool_name]}} + def format_conversation_history( self, signature: type[Signature], @@ -1277,25 +1329,10 @@ def format_conversation_history( if conversation_history is None: return [] - messages = [] - for message in conversation_history: - messages.append( - { - "role": "user", - "content": self.format_user_message_content(signature, message), - } - ) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content(signature, message), - } - ) - # Remove the history field from the inputs del inputs[history_field_name] - - return messages + history = History(messages=conversation_history) + return [message_to_openai_chat(message) for message in self.format_history(history, signature)] def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: """Parse the LM output into a dictionary of the output fields. From f756df6c6e9924f1d261c8c69aa504df57c1e241 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:31:33 -0400 Subject: [PATCH 19/23] feat(adapter): render type features as LM parts --- dspy/adapters/base.py | 77 ++------ dspy/adapters/chat_adapter.py | 91 ++++++--- dspy/adapters/json_adapter.py | 114 ++++++++--- dspy/adapters/utils.py | 11 +- dspy/adapters/xml_adapter.py | 49 ++++- dspy/clients/openai_format.py | 20 -- tests/adapters/test_adapter_field_parts.py | 67 +++++++ tests/adapters/test_chat_adapter.py | 215 +++++++++++++++------ tests/adapters/test_json_adapter.py | 8 +- 9 files changed, 445 insertions(+), 207 deletions(-) create mode 100644 tests/adapters/test_adapter_field_parts.py diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index d9a29e772b..599067981d 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -20,11 +20,11 @@ _AdapterCallPlan, _CallContext, _LMCapabilities, + _merge_lm_config, _OutputParser, _TypeFeatureHandler, _TypeParseContext, _TypeRenderContext, - _merge_lm_config, ) from dspy.adapters.types import History, Type from dspy.adapters.types.reasoning import Reasoning @@ -317,14 +317,6 @@ def _prepare_call_plan(self, call: _AdapterCallPlan, context: _CallContext) -> N for handler in self._type_feature_handlers(): handler.prepare(call, context) - if ( - call.render_signature is not call.source_signature - and call.source_signature.instructions == self._default_signature_instructions(call.source_signature) - ): - call.render_signature = call.render_signature.with_instructions( - self._default_signature_instructions(call.render_signature) - ) - @staticmethod def _default_signature_instructions(signature: type[Signature]) -> str: inputs = ", ".join([f"`{field}`" for field in signature.input_fields]) @@ -697,13 +689,11 @@ def render_messages( history_field_name = self._get_history_field_name(source_signature) if history_field_name: if history_field_name in render_signature.fields: - previous_render_signature = render_signature render_signature = render_signature.delete(history_field_name) - if previous_render_signature.instructions == self._default_signature_instructions( - previous_render_signature - ): + if source_signature.instructions == self._default_signature_instructions(source_signature): + instruction_signature = source_signature.delete(history_field_name) render_signature = render_signature.with_instructions( - self._default_signature_instructions(render_signature) + self._default_signature_instructions(instruction_signature) ) history_obj = inputs_copy.pop(history_field_name, None) if history_obj is not None: @@ -1192,60 +1182,16 @@ def format_demos( """ context = context or self._default_call_context(use_native_tool_calls=use_native_tool_calls) active_fields = signature.fields - complete_demos = [] - incomplete_demos = [] + messages = [] + incomplete_demo_prefix = "This is an example of the task, though some input or output fields are not supplied." for demo in demos: - # Check if all fields are present and not None is_complete = all(k in demo and demo[k] is not None for k in active_fields) - - # Check if demo has at least one input and one output field has_input = any(k in demo for k in signature.input_fields) has_output = any(k in demo for k in signature.output_fields) + if not (is_complete or (has_input and has_output)): + continue - if is_complete: - complete_demos.append(demo) - elif has_input and has_output: - # We only keep incomplete demos that have at least one input and one output field - incomplete_demos.append(demo) - - messages = [] - - incomplete_demo_prefix = "This is an example of the task, though some input or output fields are not supplied." - for demo in incomplete_demos: - demo_call = _AdapterCallPlan.from_signature( - signature, - {key: demo[key] for key in signature.input_fields if key in demo}, - context.lm_kwargs, - ) - self._prepare_call_plan(demo_call, context) - demo_inputs = { - key: demo_call.inputs[key] - for key in demo_call.render_signature.input_fields - if key in demo_call.inputs - } - demo_outputs = {key: demo[key] for key in signature.output_fields if key in demo} - messages.extend( - self._format_input_messages( - demo_call.render_signature, - demo_inputs, - main_request=False, - context=context, - prefix=incomplete_demo_prefix, - ) - ) - messages.extend( - self._format_output_messages_for_call( - demo_call, - demo_outputs, - len(messages), - context=context, - missing_field_message="Not supplied for this particular example. ", - ) - ) - messages.extend(demo_call.messages) - - for demo in complete_demos: demo_call = _AdapterCallPlan.from_signature( signature, {key: demo[key] for key in signature.input_fields if key in demo}, @@ -1264,6 +1210,7 @@ def format_demos( demo_inputs, main_request=False, context=context, + prefix="" if is_complete else incomplete_demo_prefix, ) ) messages.extend( @@ -1272,7 +1219,11 @@ def format_demos( demo_outputs, len(messages), context=context, - missing_field_message="Not supplied for this conversation history message. ", + missing_field_message=( + "Not supplied for this conversation history message. " + if is_complete + else "Not supplied for this particular example. " + ), ) ) messages.extend(demo_call.messages) diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py index 016c885d63..90779a9260 100644 --- a/dspy/adapters/chat_adapter.py +++ b/dspy/adapters/chat_adapter.py @@ -13,6 +13,7 @@ translate_field_type, ) from dspy.clients.base_lm import BaseLM +from dspy.core.types import LMOutput, LMPart, LMTextPart from dspy.signatures.signature import Signature from dspy.utils.callback import BaseCallback from dspy.utils.exceptions import AdapterParseError, ContextWindowExceededError @@ -42,6 +43,7 @@ def __init__( callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = False, native_response_types: list[type[type]] | None = None, + allow_parallel_tool_calls: bool | None = None, use_json_adapter_fallback: bool = True, ): """ @@ -49,6 +51,7 @@ def __init__( callbacks: List of callback functions to execute during adapter methods. use_native_function_calling: Whether to enable native function calling capabilities. native_response_types: List of output field types handled by native LM features. + allow_parallel_tool_calls: Whether to allow multiple tool calls in one LM turn. use_json_adapter_fallback: Whether to automatically fallback to JSONAdapter if the ChatAdapter fails. If True, when an error occurs (except ContextWindowExceededError), the adapter will retry using JSONAdapter. Defaults to True. @@ -57,6 +60,7 @@ def __init__( callbacks=callbacks, use_native_function_calling=use_native_function_calling, native_response_types=native_response_types, + allow_parallel_tool_calls=allow_parallel_tool_calls, ) self.use_json_adapter_fallback = use_json_adapter_fallback @@ -82,7 +86,12 @@ def __call__( # On context window exceeded error, already using JSONAdapter, or use_json_adapter_fallback is False # we don't want to retry with a different adapter. Raise the original error instead of the fallback error. raise e - return JSONAdapter()(lm, lm_kwargs, signature, demos, inputs) + return JSONAdapter( + callbacks=self.callbacks, + use_native_function_calling=self.use_native_function_calling, + native_response_types=self.native_response_types, + allow_parallel_tool_calls=self.allow_parallel_tool_calls, + )(lm, lm_kwargs, signature, demos, inputs) async def acall( self, @@ -106,13 +115,18 @@ async def acall( # On context window exceeded error, already using JSONAdapter, or use_json_adapter_fallback is False # we don't want to retry with a different adapter. Raise the original error instead of the fallback error. raise e - return await JSONAdapter().acall(lm, lm_kwargs, signature, demos, inputs) + return await JSONAdapter( + callbacks=self.callbacks, + use_native_function_calling=self.use_native_function_calling, + native_response_types=self.native_response_types, + allow_parallel_tool_calls=self.allow_parallel_tool_calls, + ).acall(lm, lm_kwargs, signature, demos, inputs) def format_field_description(self, signature: type[Signature]) -> str: - return ( - f"Your input fields are:\n{get_field_description_string(signature.input_fields)}\n" - f"Your output fields are:\n{get_field_description_string(signature.output_fields)}" - ) + description = f"Your input fields are:\n{get_field_description_string(signature.input_fields)}" + if signature.output_fields: + description += f"\nYour output fields are:\n{get_field_description_string(signature.output_fields)}" + return description def format_field_structure(self, signature: type[Signature]) -> str: """ @@ -132,8 +146,9 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo]): ) parts.append(format_signature_fields_for_instructions(signature.input_fields)) - parts.append(format_signature_fields_for_instructions(signature.output_fields)) - parts.append("[[ ## completed ## ]]\n") + if signature.output_fields: + parts.append(format_signature_fields_for_instructions(signature.output_fields)) + parts.append("[[ ## completed ## ]]\n") return "\n\n".join(parts).strip() def format_task_description(self, signature: type[Signature]) -> str: @@ -164,7 +179,7 @@ def format_user_message_content( messages.append(suffix) return "\n\n".join(messages).strip() - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: """Returns a simplified format reminder for the language model. In chat-based interactions, language models may lose track of the required output format @@ -181,6 +196,8 @@ def user_message_output_requirements(self, signature: type[Signature]) -> str: This is a more lightweight version of `format_field_structure` specifically designed for inline reminders within chat messages. """ + if not signature.output_fields: + return None def type_info(v): if v.annotation is not str: @@ -209,20 +226,7 @@ def format_assistant_message_content( return assistant_message_content def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: - sections = [(None, [])] - - for line in completion.splitlines(): - match = field_header_pattern.match(line.strip()) - if match: - # If the header pattern is found, split the rest of the line as content - header = match.group(1) - remaining_content = line[match.end() :].strip() - sections.append((header, [remaining_content] if remaining_content else [])) - else: - sections[-1][1].append(line) - - sections = [(k, "\n".join(v).strip()) for k, v in sections] - + sections = self._split_field_sections(completion) fields = {} for k, v in sections: if (k not in fields) and (k in signature.output_fields): @@ -245,6 +249,47 @@ def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: return fields + def parse_output_fields_to_parts( + self, + output: LMOutput, + signature: type[Signature], + ) -> dict[str, list[LMPart]]: + completion = output.text or "" + sections = self._split_field_sections(completion) + fields: dict[str, list[LMPart]] = {} + for key, value in sections: + if key not in fields and key in signature.output_fields: + fields[key] = [LMTextPart(text=value)] + if fields.keys() != signature.output_fields.keys(): + raise AdapterParseError( + adapter_name="ChatAdapter", + signature=signature, + lm_response=completion, + parsed_result={key: value[0].text if value else "" for key, value in fields.items()}, + ) + return fields + + def wrap_input_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + return [LMTextPart(text=f"[[ ## {field_name} ## ]]\n"), *parts] + + def wrap_output_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + return [LMTextPart(text=f"[[ ## {field_name} ## ]]\n"), *parts] + + def _split_field_sections(self, completion: str) -> list[tuple[str | None, str]]: + sections = [(None, [])] + + for line in completion.splitlines(): + match = field_header_pattern.match(line.strip()) + if match: + # If the header pattern is found, split the rest of the line as content + header = match.group(1) + remaining_content = line[match.end() :].strip() + sections.append((header, [remaining_content] if remaining_content else [])) + else: + sections[-1][1].append(line) + + return [(k, "\n".join(v).strip()) for k, v in sections] + def format_field_with_value(self, fields_with_values: dict[FieldInfoWithName, Any]) -> str: """ Formats the values of the specified fields according to the field's DSPy type (input or output), diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py index 59a3b6f563..6c4182c74e 100644 --- a/dspy/adapters/json_adapter.py +++ b/dspy/adapters/json_adapter.py @@ -7,6 +7,7 @@ import regex from pydantic.fields import FieldInfo +from dspy.adapters._type_runtime import _CallContext from dspy.adapters.chat_adapter import ChatAdapter, FieldInfoWithName from dspy.adapters.types.tool import ToolCalls from dspy.adapters.utils import ( @@ -17,6 +18,7 @@ translate_field_type, ) from dspy.clients.base_lm import BaseLM +from dspy.core.types import LMOutput, LMPart, LMTextPart from dspy.signatures.signature import Signature, SignatureMeta from dspy.utils.callback import BaseCallback from dspy.utils.exceptions import AdapterParseError @@ -38,9 +40,20 @@ def _has_open_ended_mapping(signature: SignatureMeta) -> bool: class JSONAdapter(ChatAdapter): - def __init__(self, callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = True): + def __init__( + self, + callbacks: list[BaseCallback] | None = None, + use_native_function_calling: bool = True, + native_response_types: list[type[type]] | None = None, + allow_parallel_tool_calls: bool | None = None, + ): # JSONAdapter uses native function calling by default. - super().__init__(callbacks=callbacks, use_native_function_calling=use_native_function_calling) + super().__init__( + callbacks=callbacks, + use_native_function_calling=use_native_function_calling, + native_response_types=native_response_types, + allow_parallel_tool_calls=allow_parallel_tool_calls, + ) def _json_adapter_call_common(self, lm, lm_kwargs, signature, demos, inputs, call_fn): """Common call logic to be used for both sync and async calls.""" @@ -68,10 +81,6 @@ def __call__( return result try: - structured_output_model = _get_structured_outputs_response_format( - signature, self.use_native_function_calling - ) - lm_kwargs["response_format"] = structured_output_model return super().__call__(lm, lm_kwargs, signature, demos, inputs) except Exception: logger.warning("Failed to use structured output format, falling back to JSON mode.") @@ -91,16 +100,38 @@ async def acall( return await result try: - structured_output_model = _get_structured_outputs_response_format( - signature, self.use_native_function_calling - ) - lm_kwargs["response_format"] = structured_output_model return await super().acall(lm, lm_kwargs, signature, demos, inputs) except Exception: logger.warning("Failed to use structured output format, falling back to JSON mode.") lm_kwargs["response_format"] = {"type": "json_object"} return await super().acall(lm, lm_kwargs, signature, demos, inputs) + def _prepare_request_kwargs( + self, + lm: BaseLM, + lm_kwargs: dict[str, Any], + source_signature: type[Signature], + render_signature: type[Signature], + context: _CallContext, + ) -> dict[str, Any]: + request_kwargs = dict(lm_kwargs) + if "response_format" in request_kwargs or "response_format" not in lm.supported_params: + return request_kwargs + + has_tool_calls = any(field.annotation == ToolCalls for field in source_signature.output_fields.values()) + if ( + _has_open_ended_mapping(render_signature) + or (not self.use_native_function_calling and has_tool_calls) + or not lm.supports_response_schema + ): + request_kwargs["response_format"] = {"type": "json_object"} + return request_kwargs + + request_kwargs["response_format"] = _get_structured_outputs_response_format( + render_signature, self.use_native_function_calling + ) + return request_kwargs + def format_field_structure(self, signature: type[Signature]) -> str: parts = [] parts.append("All interactions will be structured in the following way, with the appropriate values filled in.") @@ -116,11 +147,15 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo], role: parts.append("Inputs will have the following structure:") parts.append(format_signature_fields_for_instructions(signature.input_fields, role="user")) - parts.append("Outputs will be a JSON object with the following fields.") - parts.append(format_signature_fields_for_instructions(signature.output_fields, role="assistant")) + if signature.output_fields: + parts.append("Outputs will be a JSON object with the following fields.") + parts.append(format_signature_fields_for_instructions(signature.output_fields, role="assistant")) return "\n\n".join(parts).strip() - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: + if not signature.output_fields: + return None + def type_info(v): return ( f" (must be formatted as a valid Python {get_annotation_name(v.annotation)})" @@ -146,30 +181,32 @@ def format_assistant_message_content( return self.format_field_with_value(fields_with_values, role="assistant") def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: - fields = json_repair.loads(completion) + fields = self._parse_json_fields(completion, signature) + fields = {k: v for k, v in fields.items() if k in signature.output_fields} - if not isinstance(fields, dict): - pattern = r"\{(?:[^{}]|(?R))*\}" - match = regex.search(pattern, completion, regex.DOTALL) - if match: - completion = match.group(0) - fields = json_repair.loads(completion) + # Attempt to cast each value to type signature.output_fields[k].annotation. + for k, v in fields.items(): + if k in signature.output_fields: + fields[k] = parse_value(v, signature.output_fields[k].annotation) - if not isinstance(fields, dict): + if fields.keys() != signature.output_fields.keys(): raise AdapterParseError( adapter_name="JSONAdapter", signature=signature, lm_response=completion, - message="LM response cannot be serialized to a JSON object.", + parsed_result=fields, ) - fields = {k: v for k, v in fields.items() if k in signature.output_fields} - - # Attempt to cast each value to type signature.output_fields[k].annotation. - for k, v in fields.items(): - if k in signature.output_fields: - fields[k] = parse_value(v, signature.output_fields[k].annotation) + return fields + def parse_output_fields_to_parts( + self, + output: LMOutput, + signature: type[Signature], + ) -> dict[str, list[LMPart]]: + completion = output.text or "" + fields = self._parse_json_fields(completion, signature) + fields = {k: v for k, v in fields.items() if k in signature.output_fields} if fields.keys() != signature.output_fields.keys(): raise AdapterParseError( adapter_name="JSONAdapter", @@ -177,7 +214,28 @@ def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: lm_response=completion, parsed_result=fields, ) + return { + key: [LMTextPart(text=value if isinstance(value, str) else json.dumps(value, ensure_ascii=False))] + for key, value in fields.items() + } + + def _parse_json_fields(self, completion: str, signature: type[Signature]) -> dict[str, Any]: + fields = json_repair.loads(completion) + if not isinstance(fields, dict): + pattern = r"\{(?:[^{}]|(?R))*\}" + match = regex.search(pattern, completion, regex.DOTALL) + if match: + completion = match.group(0) + fields = json_repair.loads(completion) + + if not isinstance(fields, dict): + raise AdapterParseError( + adapter_name="JSONAdapter", + signature=signature, + lm_response=completion, + message="LM response cannot be serialized to a JSON object.", + ) return fields def format_field_with_value(self, fields_with_values: dict[FieldInfoWithName, Any], role: str = "user") -> str: diff --git a/dspy/adapters/utils.py b/dspy/adapters/utils.py index b45eda5593..2dd1f0a62e 100644 --- a/dspy/adapters/utils.py +++ b/dspy/adapters/utils.py @@ -14,6 +14,7 @@ from dspy.adapters.types.base_type import Type as DspyType from dspy.adapters.types.code import Code from dspy.adapters.types.reasoning import Reasoning +from dspy.adapters.types.tool import ToolCalls from dspy.signatures.utils import get_dspy_field_type @@ -74,7 +75,7 @@ def format_field_value(field_info: FieldInfo, value: Any, assume_text=True) -> s return {"type": "text", "text": string_value} -def _get_json_schema(field_type): +def _get_json_schema(field_type, field_info=None): def move_type_to_front(d): # Move the 'type' key to the front of the dictionary, recursively, for LLM readability/adherence. if isinstance(d, Mapping): @@ -85,7 +86,10 @@ def move_type_to_front(d): return [move_type_to_front(item) for item in d] return d - schema = pydantic.TypeAdapter(field_type).json_schema() + if field_type is ToolCalls: + schema = ToolCalls.json_schema(max_items=ToolCalls.max_items_from_field_info(field_info)) + else: + schema = pydantic.TypeAdapter(field_type).json_schema() schema = move_type_to_front(schema) return schema @@ -112,7 +116,8 @@ def translate_field_type(field_name, field_info): # Code has a rich type description already; avoid duplicating its large schema block. desc = "" else: - desc = f"must adhere to the JSON schema: {json.dumps(_get_json_schema(field_type), ensure_ascii=False)}" + json_schema = json.dumps(_get_json_schema(field_type, field_info), ensure_ascii=False) + desc = f"must adhere to the JSON schema: {json_schema}" desc = (" " * 8) + f"# note: the value you produce {desc}" if desc else "" return f"{{{field_name}}}{desc}" diff --git a/dspy/adapters/xml_adapter.py b/dspy/adapters/xml_adapter.py index 662b1d2868..d835c04c8d 100644 --- a/dspy/adapters/xml_adapter.py +++ b/dspy/adapters/xml_adapter.py @@ -5,6 +5,7 @@ from dspy.adapters.chat_adapter import ChatAdapter, FieldInfoWithName from dspy.adapters.utils import format_field_value, translate_field_type +from dspy.core.types import LMOutput, LMPart, LMTextPart from dspy.signatures.signature import Signature from dspy.utils.callback import BaseCallback @@ -38,7 +39,8 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo]): ) parts.append(format_signature_fields_for_instructions(signature.input_fields)) - parts.append(format_signature_fields_for_instructions(signature.output_fields)) + if signature.output_fields: + parts.append(format_signature_fields_for_instructions(signature.output_fields)) return "\n\n".join(parts).strip() def format_user_message_content( @@ -79,19 +81,17 @@ def format_assistant_message_content( }, ) - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: + if not signature.output_fields: + return None + message = "Respond with the corresponding output fields wrapped in XML tags " message += ", then ".join(f"`<{f}>`" for f in signature.output_fields) message += "." return message def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: - fields = {} - for match in self.field_pattern.finditer(completion): - name = match.group("name") - content = match.group("content").strip() - if name in signature.output_fields and name not in fields: - fields[name] = content + fields = self._parse_xml_fields(completion, signature) # Cast values using base class parse_value helper for k, v in fields.items(): fields[k] = self._parse_field_value(signature.output_fields[k], v, completion, signature) @@ -106,6 +106,39 @@ def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: ) return fields + def parse_output_fields_to_parts( + self, + output: LMOutput, + signature: type[Signature], + ) -> dict[str, list[LMPart]]: + completion = output.text or "" + fields = self._parse_xml_fields(completion, signature) + if fields.keys() != signature.output_fields.keys(): + from dspy.utils.exceptions import AdapterParseError + + raise AdapterParseError( + adapter_name="XMLAdapter", + signature=signature, + lm_response=completion, + parsed_result=fields, + ) + return {key: [LMTextPart(text=value)] for key, value in fields.items()} + + def wrap_input_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + return [LMTextPart(text=f"<{field_name}>\n"), *parts, LMTextPart(text=f"\n")] + + def wrap_output_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + return [LMTextPart(text=f"<{field_name}>\n"), *parts, LMTextPart(text=f"\n")] + + def _parse_xml_fields(self, completion: str, signature: type[Signature]) -> dict[str, str]: + fields = {} + for match in self.field_pattern.finditer(completion): + name = match.group("name") + content = match.group("content").strip() + if name in signature.output_fields and name not in fields: + fields[name] = content + return fields + def _parse_field_value(self, field_info, raw, completion, signature): from dspy.adapters.utils import parse_value diff --git a/dspy/clients/openai_format.py b/dspy/clients/openai_format.py index 98d72d5a2d..4fd88ece5b 100644 --- a/dspy/clients/openai_format.py +++ b/dspy/clients/openai_format.py @@ -882,26 +882,6 @@ def lm_response_from_legacy_outputs(outputs: list[dict[str, Any] | str | None], return LMResponse(model=request.model, outputs=[lm_output_from_legacy_output(output) for output in outputs]) -def legacy_outputs_from_lm_response(response: LMResponse) -> list[dict[str, Any] | str | None]: - """Return legacy adapter postprocess values from a normalized response. - - This is a temporary compatibility helper for current adapter postprocessing. - Prefer the exact original legacy output when it is available so compatibility - hooks that distinguish `str` from provider dictionaries keep working. - """ - outputs = [] - for output in response.outputs: - if output.metadata.get("empty_legacy_outputs"): - continue - if output.provider_output is not None: - outputs.append(output.provider_output) - elif output.text is not None and not output.reasoning_content and not output.tool_calls and not output.citations and output.logprobs is None: - outputs.append(output.text) - else: - outputs.append(output.to_output_dict()) - return outputs - - def lm_output_from_legacy_output(output: dict[str, Any] | str | None) -> LMOutput: """Normalize one current legacy `BaseLM` output item into an `LMOutput`.""" if isinstance(output, str): diff --git a/tests/adapters/test_adapter_field_parts.py b/tests/adapters/test_adapter_field_parts.py new file mode 100644 index 0000000000..01a625de39 --- /dev/null +++ b/tests/adapters/test_adapter_field_parts.py @@ -0,0 +1,67 @@ +import dspy +from dspy.core.types import LMOutput, LMTextPart + + +def test_base_value_helpers_round_trip_text_field_parts(): + class SimpleSignature(dspy.Signature): + answer: int = dspy.OutputField() + + adapter = dspy.ChatAdapter() + field_info = SimpleSignature.output_fields["answer"] + + parts = adapter.value_to_lm_parts(42, field_info) + + assert parts == [LMTextPart(text="42")] + assert adapter.lm_parts_to_value(parts, field_info) == 42 + + +def test_chat_adapter_wraps_and_slices_field_parts(): + class SimpleSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.ChatAdapter() + + wrapped = adapter.wrap_input_field_parts("question", [LMTextPart(text="What?")]) + assert wrapped == [LMTextPart(text="[[ ## question ## ]]\n"), LMTextPart(text="What?")] + + output = LMOutput(parts=[LMTextPart(text="[[ ## answer ## ]]\nParis\n\n[[ ## completed ## ]]")]) + fields = adapter.parse_output_fields_to_parts(output, SimpleSignature) + + assert fields == {"answer": [LMTextPart(text="Paris")]} + + +def test_json_adapter_slices_output_fields_to_text_parts(): + class SimpleSignature(dspy.Signature): + answer: str = dspy.OutputField() + score: float = dspy.OutputField() + + adapter = dspy.JSONAdapter() + output = LMOutput(parts=[LMTextPart(text='{"answer": "Paris", "score": 0.9}')]) + + fields = adapter.parse_output_fields_to_parts(output, SimpleSignature) + + assert fields == { + "answer": [LMTextPart(text="Paris")], + "score": [LMTextPart(text="0.9")], + } + + +def test_xml_adapter_wraps_and_slices_field_parts(): + class SimpleSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.XMLAdapter() + + wrapped = adapter.wrap_output_field_parts("answer", [LMTextPart(text="Paris")]) + assert wrapped == [ + LMTextPart(text="\n"), + LMTextPart(text="Paris"), + LMTextPart(text="\n"), + ] + + output = LMOutput(parts=[LMTextPart(text="Paris")]) + fields = adapter.parse_output_fields_to_parts(output, SimpleSignature) + + assert fields == {"answer": [LMTextPart(text="Paris")]} diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py index ef2cc38a5e..bdc2b73432 100644 --- a/tests/adapters/test_chat_adapter.py +++ b/tests/adapters/test_chat_adapter.py @@ -350,16 +350,12 @@ class HistorySignature(dspy.Signature): expected_messages = [{"role": "system", "content": "Your input fields are:\n" - "1. `history` (History): \n" - "2. `question` (str):\n" + "1. `question` (str):\n" "Your output fields are:\n" "1. `answer` (str):\n" "All interactions will be structured in the following way, with the appropriate " "values filled in.\n" "\n" - "[[ ## history ## ]]\n" - "{history}\n" - "\n" "[[ ## question ## ]]\n" "{question}\n" "\n" @@ -602,19 +598,15 @@ class RichRenderingSignature(dspy.Signature): expected_messages = [{"role": "system", "content": 'Your input fields are:\n' - '1. `history` (History): \n' - '2. `image` (Image): \n' - '3. `tools` (list[Tool]): \n' - '4. `profile` (Profile): \n' - '5. `question` (str):\n' + '1. `image` (Image): \n' + '2. `tools` (list[Tool]): \n' + '3. `profile` (Profile): \n' + '4. `question` (str):\n' 'Your output fields are:\n' '1. `answer` (AnswerCard):\n' 'All interactions will be structured in the following way, with the appropriate ' 'values filled in.\n' '\n' - '[[ ## history ## ]]\n' - '{history}\n' - '\n' '[[ ## image ## ]]\n' '{image}\n' '\n' @@ -637,11 +629,7 @@ class RichRenderingSignature(dspy.Signature): 'In adhering to this structure, your objective is: \n' ' Answer using all supplied context.'}, {"role": "user", - "content": [{"type": "text", - "text": "This is an example of the task, though some input or output fields are not " - "supplied.\n" - "\n" - "[[ ## image ## ]]\n"}, + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, {"type": "text", "text": '\n' @@ -675,7 +663,8 @@ class RichRenderingSignature(dspy.Signature): '\n' '[[ ## completed ## ]]\n'}, {"role": "user", - "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + "content": [{"type": "text", + "text": "[[ ## image ## ]]\n"}, {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, {"type": "text", "text": '\n' @@ -1080,25 +1069,16 @@ class NativeToolSignature(dspy.Signature): expected_messages = [{"role": "system", "content": "Your input fields are:\n" "1. `question` (str):\n" - "Your output fields are:\n" - "\n" "All interactions will be structured in the following way, with the appropriate " "values filled in.\n" "\n" "[[ ## question ## ]]\n" "{question}\n" - "\n" - "\n" - "\n" - "[[ ## completed ## ]]\n" "In adhering to this structure, your objective is: \n" " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, {"role": "user", "content": "[[ ## question ## ]]\n" - "Q?\n" - "\n" - "Respond with the corresponding output fields, starting with the field , and then " - "ending with the marker for `[[ ## completed ## ]]`."}] + "Q?"}] assert messages == expected_messages expected_lm_kwargs = {"tools": [{"type": "function", "function": {"name": "search", @@ -1268,20 +1248,19 @@ class KitchenSinkSignature(dspy.Signature): expected_messages = [{"role": "system", "content": 'Your input fields are:\n' - '1. `history` (History): \n' - '2. `image` (Image): \n' - '3. `audio` (Audio): \n' - '4. `file` (File): \n' - '5. `document` (Document): \n' + '1. `image` (Image): \n' + '2. `audio` (Audio): \n' + '3. `file` (File): \n' + '4. `document` (Document): \n' ' Type description of Document: A document containing text content that can be ' 'referenced and cited. Include the full text content and optionally a title for ' 'proper referencing.\n' - '6. `event` (Event): \n' + '5. `event` (Event): \n' ' Type description of Event: An event block.\n' - '7. `tools` (list[Tool]): \n' - '8. `profile` (Profile): \n' - '9. `context` (str): \n' - '10. `question` (str):\n' + '6. `tools` (list[Tool]): \n' + '7. `profile` (Profile): \n' + '8. `context` (str): \n' + '9. `question` (str):\n' 'Your output fields are:\n' '1. `answer` (AnswerCard): \n' "2. `verdict` (Literal['yes', 'no']): \n" @@ -1289,9 +1268,6 @@ class KitchenSinkSignature(dspy.Signature): 'All interactions will be structured in the following way, with the appropriate ' 'values filled in.\n' '\n' - '[[ ## history ## ]]\n' - '{history}\n' - '\n' '[[ ## image ## ]]\n' '{image}\n' '\n' @@ -1336,11 +1312,7 @@ class KitchenSinkSignature(dspy.Signature): 'In adhering to this structure, your objective is: \n' ' Answer carefully using every available signal.'}, {"role": "user", - "content": [{"type": "text", - "text": "This is an example of the task, though some input or output fields are not " - "supplied.\n" - "\n" - "[[ ## image ## ]]\n"}, + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, {"type": "text", "text": "\n\n[[ ## audio ## ]]\n"}, {"type": "input_audio", "input_audio": {"data": "REVNTw==", "format": "wav"}}, @@ -1926,7 +1898,13 @@ def get_weather(city: str) -> str: ) assert result[0]["tool_calls"] == dspy.ToolCalls( - tool_calls=[dspy.ToolCalls.ToolCall(name="get_weather", args={"city": "Paris"})] + tool_calls=[ + dspy.ToolCalls.ToolCall( + name="get_weather", + args={"city": "Paris"}, + id="call_pQm8ajtSMxgA0nrzK2ivFmxG", + ) + ] ) # `answer` is not present, so we set it to None assert result[0]["answer"] is None @@ -1948,6 +1926,124 @@ def get_weather(city: str) -> str: assert result[0]["tool_calls"] is None +def test_chat_adapter_toolcalls_serial_non_native_schema_limits_cardinality(): + class MySignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + def get_weather(city: str) -> str: + return f"The weather in {city} is sunny" + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(allow_parallel_tool_calls=False), + MySignature, + [], + {"question": "What is the weather in Paris?", "tools": [dspy.Tool(get_weather)]}, + ) + + assert '"maxItems": 1' in messages[0]["content"] + assert "parallel_tool_calls" not in lm_kwargs + + +def test_chat_adapter_toolcalls_native_serial_forwards_provider_flag(): + class MySignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + def get_weather(city: str) -> str: + return f"The weather in {city} is sunny" + + adapter = dspy.ChatAdapter(use_native_function_calling=True, allow_parallel_tool_calls=False) + inputs = {"question": "What is the weather in Paris?", "tools": [dspy.Tool(get_weather)]} + lm = dspy.LM(model="openai/gpt-4o-mini", cache=False) + rendered = adapter._render_request(lm, {}, MySignature, [], inputs) + rendered_again = adapter._render_request(lm, {}, MySignature, [], inputs) + + assert rendered.request.config.tool_choice.parallel is False + assert rendered.request.config.extensions == {} + assert rendered.call_plan.tools + assert rendered.call_plan is not rendered_again.call_plan + assert not hasattr(rendered.context, "extra") + + _, lm_kwargs = format_messages_and_lm_kwargs( + adapter, + MySignature, + [], + inputs, + lm=lm, + ) + + assert lm_kwargs["parallel_tool_calls"] is False + + +def test_chat_adapter_toolcalls_native_render_uses_tool_call_path(): + class MySignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + def get_weather(city: str) -> str: + return f"The weather in {city} is sunny" + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(use_native_function_calling=True), + MySignature, + [], + {"question": "What is the weather in Paris?", "tools": [dspy.Tool(get_weather)]}, + lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), + ) + + assert "[[ ## tool_calls ## ]]" not in messages[0]["content"] + assert "[[ ## tools ## ]]" not in messages[0]["content"] + assert lm_kwargs["tools"][0]["function"]["name"] == "get_weather" + + +def test_chat_adapter_toolcalls_serial_non_native_rejects_multiple_calls(): + class MySignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + def get_weather(city: str) -> str: + return f"The weather in {city} is sunny" + + def get_time(city: str) -> str: + return f"The time in {city} is noon" + + adapter = dspy.ChatAdapter(allow_parallel_tool_calls=False, use_json_adapter_fallback=False) + with mock.patch("litellm.completion") as mock_completion: + mock_completion.return_value = ModelResponse( + choices=[ + Choices( + message=Message( + content=( + "[[ ## tool_calls ## ]]\n" + "[" + "{'name': 'get_weather', 'args': {'city': 'Paris'}}, " + "{'name': 'get_time', 'args': {'city': 'Paris'}}" + "]" + ) + ) + ) + ], + model="openai/gpt-4o-mini", + ) + + with pytest.raises(ValueError, match="permits at most 1"): + adapter( + dspy.LM(model="openai/gpt-4o-mini", cache=False), + {}, + MySignature, + [], + { + "question": "What is the weather and time in Paris?", + "tools": [dspy.Tool(get_weather), dspy.Tool(get_time)], + }, + ) + + def test_chat_adapter_toolcalls_vague_match(): class MySignature(dspy.Signature): question: str = dspy.InputField() @@ -2028,13 +2124,14 @@ class MySignature(dspy.Signature): ], model="anthropic/claude-3-7-sonnet-20250219", ) - modified_signature = adapter._call_preprocess( - dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), - {}, + messages, _ = format_messages_and_lm_kwargs( + adapter, MySignature, + [], {"question": "What is the capital of France?"}, + lm=dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), ) - assert "reasoning" not in modified_signature.output_fields + assert "[[ ## reasoning ## ]]" not in messages[0]["content"] result = adapter( dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), @@ -2145,15 +2242,23 @@ def test_empty_string_content_raises_adapter_parse_error(): def test_tool_call_with_null_content_does_not_raise(): - """Tool-call-only responses legitimately have content=None. - _call_postprocess must NOT raise when tool_calls are present.""" + """Tool-call-only responses legitimately have content=None and should parse.""" + from dspy.clients.openai_format import lm_response_from_legacy_outputs + adapter = dspy.ChatAdapter(use_native_function_calling=True) sig_cls = dspy.Signature("question, tools: list[dspy.Tool] -> answer, tool_calls: dspy.ToolCalls") + def search(query: str) -> str: + return query + + inputs = {"question": "test", "tools": [dspy.Tool(search)]} + lm = dspy.LM("openai/gpt-4o-mini", cache=False) + rendered = adapter._render_request(lm, {}, sig_cls, [], inputs) outputs = [{"text": None, "tool_calls": [ {"function": {"name": "search", "arguments": '{"query": "test"}'}, "id": "call_1", "type": "function"} ]}] + response = lm_response_from_legacy_outputs(outputs, rendered.request) - result = adapter._call_postprocess(sig_cls, sig_cls, outputs, None, {}) + result = adapter._parse_response(sig_cls, rendered.call_plan, response, rendered.context) assert result is not None assert len(result) == 1 diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py index 2cd7f2c5a5..e0f9851334 100644 --- a/tests/adapters/test_json_adapter.py +++ b/tests/adapters/test_json_adapter.py @@ -568,8 +568,6 @@ class NativeToolSignature(dspy.Signature): expected_messages = [{"role": "system", "content": "Your input fields are:\n" "1. `question` (str):\n" - "Your output fields are:\n" - "\n" "All interactions will be structured in the following way, with the appropriate " "values filled in.\n" "\n" @@ -577,12 +575,8 @@ class NativeToolSignature(dspy.Signature): "\n" "[[ ## question ## ]]\n" "{question}\n" - "\n" - "Outputs will be a JSON object with the following fields.\n" - "\n" - "{}\n" "In adhering to this structure, your objective is: \n" - " Given the fields `question`, follow the instructions."}, + " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, {"role": "user", "content": "[[ ## question ## ]]\n" "Q?"}] From 2e9be53f9ea3d7ffea2ac97a31b4fc234aaa12bf Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:31:45 -0400 Subject: [PATCH 20/23] feat(adapter): normalize history values --- dspy/adapters/types/history.py | 148 +++++++++++++--------- tests/adapters/test_history.py | 115 +++++++++++++++++ tests/adapters/test_history_formatting.py | 72 +++++++++++ 3 files changed, 278 insertions(+), 57 deletions(-) create mode 100644 tests/adapters/test_history.py create mode 100644 tests/adapters/test_history_formatting.py diff --git a/dspy/adapters/types/history.py b/dspy/adapters/types/history.py index 6dda4f9b7c..1c5cbd3360 100644 --- a/dspy/adapters/types/history.py +++ b/dspy/adapters/types/history.py @@ -1,68 +1,102 @@ -from typing import Any +from typing import Any, Callable import pydantic +from pydantic import Field, model_validator +from dspy.adapters.types.tool import ToolCallResults, ToolCalls -class History(pydantic.BaseModel): - """Class representing the conversation history. - - The conversation history is a list of messages, each message entity should have keys from the associated signature. - For example, if you have the following signature: - - ``` - class MySignature(dspy.Signature): - question: str = dspy.InputField() - history: dspy.History = dspy.InputField() - answer: str = dspy.OutputField() - ``` - - Then the history should be a list of dictionaries with keys "question" and "answer". - - Examples: - ``` - import dspy - - dspy.configure(lm=dspy.LM("openai/gpt-4o-mini")) - - class MySignature(dspy.Signature): - question: str = dspy.InputField() - history: dspy.History = dspy.InputField() - answer: str = dspy.OutputField() - history = dspy.History( - messages=[ - {"question": "What is the capital of France?", "answer": "Paris"}, - {"question": "What is the capital of Germany?", "answer": "Berlin"}, - ] - ) - - predict = dspy.Predict(MySignature) - outputs = predict(question="What is the capital of France?", history=history) - ``` - - Example of capturing the conversation history: - ``` - import dspy - - dspy.configure(lm=dspy.LM("openai/gpt-4o-mini")) - - class MySignature(dspy.Signature): - question: str = dspy.InputField() - history: dspy.History = dspy.InputField() - answer: str = dspy.OutputField() - - predict = dspy.Predict(MySignature) - outputs = predict(question="What is the capital of France?") - history = dspy.History(messages=[{"question": "What is the capital of France?", **outputs}]) - outputs_with_history = predict(question="Are you sure?", history=history) - ``` - """ +class History(pydantic.BaseModel): + """Reusable ordered field history.""" - messages: list[dict[str, Any]] + messages: list[dict[str, Any]] = Field(default_factory=list) model_config = pydantic.ConfigDict( - frozen=True, str_strip_whitespace=True, - validate_assignment=True, extra="forbid", ) + + @model_validator(mode="after") + def _normalize_messages(self) -> "History": + self.messages = [ + self._normalize_message(message) if isinstance(message, dict) else message for message in self.messages + ] + return self + + def __init__(self, *args: Any, compact_fn: Callable[["History"], None] | None = None, **kwargs: Any): + super().__init__(*args, **kwargs) + object.__setattr__(self, "_compact_fn", compact_fn) + + @pydantic.model_serializer() + def serialize_model(self) -> dict[str, Any]: + return {"messages": [self._serialize_message(message) for message in self.messages]} + + def compact_if_needed(self) -> None: + fn = getattr(self, "_compact_fn", None) + if fn is not None: + fn(self) + + def append(self, message: dict[str, Any]) -> dict[str, Any]: + message = self._normalize_message(dict(message)) + self.messages.append(message) + return message + + def to_lm_messages(self, adapter: Any, signature: type[Any], *, use_native_tool_calls: bool = False) -> list[Any]: + return adapter.format_history(self, signature, use_native_tool_calls=use_native_tool_calls) + + @staticmethod + def _normalize_message(message: dict[str, Any]) -> dict[str, Any]: + normalized = {} + for key, value in message.items(): + if key == "tool_calls" and isinstance(value, list): + normalized[key] = ToolCalls.model_validate(value) + elif key == "tool_call_results" and isinstance(value, list): + normalized[key] = ToolCallResults.model_validate(value) + elif isinstance(value, dict) and set(value.keys()) == {"tool_calls"} and isinstance(value["tool_calls"], list): + normalized[key] = ToolCalls.model_validate(value) + elif ( + isinstance(value, dict) + and set(value.keys()) == {"tool_call_results"} + and isinstance(value["tool_call_results"], list) + ): + normalized[key] = ToolCallResults.model_validate(value) + else: + normalized[key] = value + return normalized + + @staticmethod + def _serialize_message(message: dict[str, Any]) -> dict[str, Any]: + serialized = {} + for key, value in message.items(): + if isinstance(value, (ToolCalls, ToolCallResults)): + serialized[key] = value.format() + elif hasattr(value, "model_dump"): + serialized[key] = value.model_dump() + else: + serialized[key] = value + return serialized + + +def truncate_oldest_actions(history: History, *, max_tokens: int = 200_000, keep_n: int = 3) -> None: + if len(str(history.messages)) // 4 <= max_tokens: + return + + action_indices = [ + idx + for idx, message in enumerate(history.messages) + if isinstance(message, dict) and any(isinstance(value, ToolCalls) for value in message.values()) + ] + drop_count = len(action_indices) - keep_n + if drop_count <= 0: + return + + drop_indices = set(action_indices[:drop_count]) + + history.messages[:] = [message for idx, message in enumerate(history.messages) if idx not in drop_indices] + + +def make_truncate_oldest_actions(max_tokens: int = 200_000, keep_n: int = 3) -> Callable[[History], None]: + def _compact(history: History) -> None: + truncate_oldest_actions(history, max_tokens=max_tokens, keep_n=keep_n) + + return _compact diff --git a/tests/adapters/test_history.py b/tests/adapters/test_history.py new file mode 100644 index 0000000000..fee489d468 --- /dev/null +++ b/tests/adapters/test_history.py @@ -0,0 +1,115 @@ +from dspy.adapters.types.history import History, make_truncate_oldest_actions, truncate_oldest_actions +from dspy.adapters.types.tool import ToolCallResults, ToolCalls + + +def test_messages_key_is_canonical_history_shape(): + legacy_message = {"question": "What is the capital of France?", "answer": "Paris"} + + history = History(messages=[legacy_message]) + + assert history.messages[0] == legacy_message + assert history.model_dump() == {"messages": [legacy_message]} + + +def test_append_adds_plain_messages(): + tool_calls = ToolCalls.from_dict_list([{"name": "search", "args": {"query": "hello"}, "id": "call_0"}]) + tool_results = ToolCallResults.from_dict_list( + [{"call_id": "call_0", "name": "search", "value": "result", "is_error": False}] + ) + history = History(messages=[]) + + history.append( + { + "question": "hi", + "next_thought": "search first", + "tool_calls": tool_calls, + "tool_call_results": tool_results, + "answer": "bye", + } + ) + + assert history.messages == [ + { + "question": "hi", + "next_thought": "search first", + "tool_calls": tool_calls, + "tool_call_results": tool_results, + "answer": "bye", + }, + ] + assert History.model_validate(history.model_dump()) == history + + +def test_history_normalizes_raw_tool_call_lists_on_canonical_keys(): + history = History( + messages=[ + {"tool_calls": [{"name": "search", "args": {"query": "hello"}, "id": "call_0"}]}, + {"tool_call_results": [{"call_id": "call_0", "name": "search", "value": "result"}]}, + ] + ) + + assert history.messages[0]["tool_calls"] == ToolCalls.from_dict_list( + [{"name": "search", "args": {"query": "hello"}, "id": "call_0"}] + ) + assert history.messages[1]["tool_call_results"] == ToolCallResults.from_dict_list( + [{"call_id": "call_0", "name": "search", "value": "result"}] + ) + + +def test_compact_if_needed_calls_compact_fn_with_history(): + calls = [] + history = History(messages=[], compact_fn=calls.append) + + history.compact_if_needed() + + assert calls == [history] + + +def test_truncate_oldest_actions_keeps_recent_complete_tool_turns_and_non_actions(): + history = History(messages=[{"question": "new"}, {"question": "legacy", "answer": "legacy answer"}]) + + for index in range(5): + history.append( + { + "next_thought": str(index), + "tool_calls": ToolCalls.from_dict_list( + [{"name": "search", "args": {"query": str(index)}, "id": f"call_{index}"}] + ), + "tool_call_results": ToolCallResults.from_dict_list( + [{"call_id": f"call_{index}", "name": "search", "value": f"result {index}"}] + ), + } + ) + history.append({"answer": "done"}) + + truncate_oldest_actions(history, max_tokens=0, keep_n=2) + + action_messages = [message for message in history.messages if "tool_calls" in message] + result_messages = [message for message in history.messages if "tool_call_results" in message] + assert [message["next_thought"] for message in action_messages] == ["3", "4"] + assert [message["tool_call_results"].tool_call_results[0].value for message in result_messages] == [ + "result 3", + "result 4", + ] + assert history.messages[0] == {"question": "new"} + assert history.messages[1] == {"question": "legacy", "answer": "legacy answer"} + assert history.messages[-1] == {"answer": "done"} + + +def test_make_truncate_oldest_actions_returns_compaction_fn(): + history = History(messages=[]) + for index in range(4): + history.append( + { + "next_thought": str(index), + "tool_calls": ToolCalls.from_dict_list([{"name": "search", "args": {}, "id": f"call_{index}"}]), + "tool_call_results": ToolCallResults.from_dict_list( + [{"call_id": f"call_{index}", "name": "search", "value": f"result {index}"}] + ), + } + ) + + make_truncate_oldest_actions(max_tokens=0, keep_n=1)(history) + + action_messages = [message for message in history.messages if "tool_calls" in message] + assert [message["next_thought"] for message in action_messages] == ["3"] diff --git a/tests/adapters/test_history_formatting.py b/tests/adapters/test_history_formatting.py new file mode 100644 index 0000000000..51c141e63c --- /dev/null +++ b/tests/adapters/test_history_formatting.py @@ -0,0 +1,72 @@ +import json + +import dspy +from dspy.adapters.types.history import History + + +def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + +class AgentTurn(dspy.Signature): + question: str = dspy.InputField() + history: dspy.History = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_call_results: dspy.ToolCallResults = dspy.InputField() + next_thought: str = dspy.OutputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + +def _history_with_complete_tool_turn() -> History: + history = History(messages=[]) + history.append( + { + "question": "What is 1+2?", + "next_thought": "I should add.", + "tool_calls": dspy.ToolCalls.from_dict_list([{"name": "add", "args": {"a": 1, "b": 2}}]), + "tool_call_results": dspy.ToolCallResults.from_dict_list( + [{"name": "add", "value": 3, "is_error": False}] + ), + } + ) + return history + + +def test_chat_adapter_formats_complete_tool_turn_without_duplicate_input(): + messages = dspy.ChatAdapter().format( + AgentTurn, + [], + {"history": _history_with_complete_tool_turn(), "tools": [dspy.Tool(add)]}, + ) + + texts = [message["content"] or "" for message in messages] + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "user", "user"] + assert sum("What is 1+2?" in text for text in texts[1:]) == 1 + assert "[[ ## next_thought ## ]]\nI should add." in texts[2] + assert "[[ ## tool_calls ## ]]" in texts[2] + assert "[[ ## tool_call_results ## ]]" in texts[3] + assert '"name": "add"' in texts[3] + assert '"value": 3' in texts[3] + assert "Respond with the corresponding output fields" in texts[4] + + +def test_json_adapter_formats_complete_tool_turn_without_duplicate_input(): + messages = dspy.JSONAdapter().format( + AgentTurn, + [], + {"history": _history_with_complete_tool_turn(), "tools": [dspy.Tool(add)]}, + ) + + texts = [message["content"] or "" for message in messages] + assistant = json.loads(texts[2]) + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "user", "user"] + assert sum("What is 1+2?" in text for text in texts[1:]) == 1 + assert assistant["next_thought"] == "I should add." + assert assistant["tool_calls"]["tool_calls"][0]["name"] == "add" + assert "[[ ## tool_call_results ## ]]" in texts[3] + assert '"name": "add"' in texts[3] + assert '"value": 3' in texts[3] + assert "Respond with a JSON object" in texts[4] From fcce0eaa15f2afee8bf6dfd68dee15d1fb1099b2 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:32:43 -0400 Subject: [PATCH 21/23] refactor(adapter): update two-step and streaming hooks --- dspy/adapters/two_step_adapter.py | 122 +++++++++++++++--------- dspy/predict/predict.py | 27 ++++-- dspy/streaming/streaming_listener.py | 4 +- tests/adapters/test_two_step_adapter.py | 38 ++++++++ tests/streaming/test_streaming.py | 21 ++-- 5 files changed, 152 insertions(+), 60 deletions(-) diff --git a/dspy/adapters/two_step_adapter.py b/dspy/adapters/two_step_adapter.py index 0e427dea25..8908bb063f 100644 --- a/dspy/adapters/two_step_adapter.py +++ b/dspy/adapters/two_step_adapter.py @@ -1,12 +1,12 @@ from typing import Any -import json_repair - from dspy.adapters.base import Adapter from dspy.adapters.chat_adapter import ChatAdapter +from dspy.adapters._type_runtime import _CallContext from dspy.adapters.types import ToolCalls from dspy.adapters.utils import get_field_description_string from dspy.clients.base_lm import BaseLM +from dspy.core.types import LMMessage from dspy.signatures.field import InputField from dspy.signatures.signature import Signature, make_signature @@ -46,7 +46,15 @@ def __init__(self, extraction_model: BaseLM, **kwargs): self.extraction_model = extraction_model def format( - self, signature: type[Signature], demos: list[dict[str, Any]], inputs: dict[str, Any] + self, + signature: type[Signature], + demos: list[dict[str, Any]], + inputs: dict[str, Any], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + source_signature: type[Signature] | None = None, + prepared_messages: list[LMMessage] | None = None, ) -> list[dict[str, Any]]: """ Format a prompt for the first stage with the main LM. @@ -67,13 +75,44 @@ def format( task_description = self.format_task_description(signature) messages.append({"role": "system", "content": task_description}) - messages.extend(self.format_demos(signature, demos)) + messages.extend( + self.format_demos( + signature, + demos, + use_native_tool_calls=use_native_tool_calls, + context=context, + ) + ) + messages.extend(prepared_messages or []) # Format the current input messages.append({"role": "user", "content": self.format_user_message_content(signature, inputs)}) return messages + def render_messages( + self, + signature: type[Signature], + demos: list[dict[str, Any]], + inputs: dict[str, Any], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + source_signature: type[Signature] | None = None, + prepared_messages: list[LMMessage] | None = None, + ) -> list[LMMessage]: + return self._coerce_lm_messages( + self.format( + signature, + demos, + inputs, + use_native_tool_calls=use_native_tool_calls, + context=context, + source_signature=source_signature, + prepared_messages=prepared_messages, + ) + ) + def parse(self, signature: Signature, completion: str) -> dict[str, Any]: """ Use a smaller LM (extraction_model) with chat adapter to extract structured data @@ -111,51 +150,47 @@ async def acall( demos: list[dict[str, Any]], inputs: dict[str, Any], ) -> list[dict[str, Any]]: - inputs = self.format(signature, demos, inputs) - - outputs = await lm.acall(messages=inputs, **lm_kwargs) - # The signature is supposed to be "text -> {original output fields}" - extractor_signature = self._create_extractor_signature(signature) + rendered = self._render_request(lm, lm_kwargs, signature, demos, inputs) + response = await self._acall_lm(lm, rendered.request) values = [] - tool_call_output_field_name = self._get_tool_call_output_field_name(signature) - for output in outputs: - output_logprobs = None - tool_calls = None - text = output - - if isinstance(output, dict): - text = output["text"] - output_logprobs = output.get("logprobs") - tool_calls = output.get("tool_calls") - - try: - # Call the smaller LM to extract structured data from the raw completion text with ChatAdapter - value = await ChatAdapter().acall( - lm=self.extraction_model, - lm_kwargs={}, - signature=extractor_signature, - demos=[], - inputs={"text": text}, + extractor_signature = self._create_extractor_signature(rendered.signature) + + for output in response.outputs: + if output.metadata.get("empty_legacy_outputs"): + continue + + has_text_output = bool(output.text and rendered.signature.output_fields) + has_tool_output = bool(output.tool_calls and tool_call_output_field_name) + + if has_text_output: + try: + value = await ChatAdapter().acall( + lm=self.extraction_model, + lm_kwargs={}, + signature=extractor_signature, + demos=[], + inputs={"text": output.text}, + ) + value = value[0] + for field_name in signature.output_fields: + value.setdefault(field_name, None) + except Exception as e: + raise ValueError(f"Failed to parse response from the original completion: {output}") from e + else: + value = dict.fromkeys(signature.output_fields.keys()) + + if has_tool_output: + value[tool_call_output_field_name] = ToolCalls.from_dict_list( + [{"name": call.name, "args": call.args, "id": call.id} for call in output.tool_calls] ) - value = value[0] - - except Exception as e: - raise ValueError(f"Failed to parse response from the original completion: {output}") from e - if tool_calls and tool_call_output_field_name: - tool_calls = [ - { - "name": v["function"]["name"], - "args": json_repair.loads(v["function"]["arguments"]), - } - for v in tool_calls - ] - value[tool_call_output_field_name] = ToolCalls.from_dict_list(tool_calls) + if not (has_text_output or has_tool_output): + raise ValueError(f"Failed to parse response from the original completion: {output}") - if output_logprobs is not None: - value["logprobs"] = output_logprobs + if output.logprobs is not None: + value["logprobs"] = output.logprobs values.append(value) return values @@ -180,6 +215,7 @@ def format_user_message_content( inputs: dict[str, Any], prefix: str = "", suffix: str = "", + main_request: bool = False, ) -> str: parts = [prefix] diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py index bd790a6f32..8b10493156 100644 --- a/dspy/predict/predict.py +++ b/dspy/predict/predict.py @@ -7,6 +7,7 @@ from pydantic_core import PydanticUndefined from dspy.adapters.chat_adapter import ChatAdapter +from dspy.adapters.types.history import History from dspy.clients.base_lm import BaseLM from dspy.clients.lm import LM from dspy.dsp.utils.settings import settings @@ -40,6 +41,18 @@ def _sanitize_lm_state(lm_state: dict, allow_unsafe_lm_state: bool) -> dict: return sanitized_lm_state +def _field_is_present_in_history(signature: type[Signature], inputs: dict[str, Any], field_name: str) -> bool: + for history_name, field in signature.input_fields.items(): + if field.annotation != History: + continue + history = inputs.get(history_name) + if not isinstance(history, History): + continue + if any(isinstance(message, dict) and field_name in message for message in history.messages): + return True + return False + + class Predict(Module, Parameter): """Basic DSPy module that maps inputs to outputs using a language model. @@ -216,11 +229,13 @@ def _forward_preprocess(self, **kwargs): if not all(k in kwargs for k in signature.input_fields): present = [k for k in signature.input_fields if k in kwargs] missing = [k for k in signature.input_fields if k not in kwargs] - logger.warning( - "Not all input fields were provided to module. Present: %s. Missing: %s.", - present, - missing, - ) + missing = [field_name for field_name in missing if not _field_is_present_in_history(signature, kwargs, field_name)] + if missing: + logger.warning( + "Not all input fields were provided to module. Present: %s. Missing: %s.", + present, + missing, + ) return lm, config, signature, demos, kwargs def _forward_postprocess(self, completions, signature, **kwargs): @@ -354,7 +369,7 @@ def _check_type(value: Any, expected: type) -> bool: return all(_check_type(item, args[0]) for item in value) if len(value) != len(args): return False - return all(_check_type(item, arg) for item, arg in zip(value, args)) + return all(_check_type(item, arg) for item, arg in zip(value, args, strict=True)) return True # set / frozenset diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.py index 754cbf54de..f5376ce5cd 100644 --- a/dspy/streaming/streaming_listener.py +++ b/dspy/streaming/streaming_listener.py @@ -145,13 +145,15 @@ def receive(self, chunk: ModelResponseStream): and issubclass(self._output_type, Type) and self._output_type.is_streamable() ): - if parsed_chunk := self._output_type.parse_stream_chunk(chunk): + parsed_chunk = self._output_type.parse_stream_chunk(chunk) + if parsed_chunk is not None: return StreamResponse( self.predict_name, self.signature_field_name, parsed_chunk, is_last_chunk=self.stream_end, ) + return None # For non-custom streamable types, the streaming chunks come from the content field of the ModelResponseStream. try: diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.py index 6eaee233a4..9bb5843fcc 100644 --- a/tests/adapters/test_two_step_adapter.py +++ b/tests/adapters/test_two_step_adapter.py @@ -189,6 +189,44 @@ class TestSignature(dspy.Signature): assert "text from main LM" in content +@pytest.mark.asyncio +async def test_two_step_adapter_async_preserves_native_tool_call_ids(): + def add(a: int, b: int) -> int: + return a + b + + class ToolSignature(dspy.Signature): + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + async def acall(self, messages, **kwargs): + return [ + { + "text": None, + "tool_calls": [ + { + "id": "call_add", + "type": "function", + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + } + ], + } + ] + + adapter = dspy.TwoStepAdapter(dspy.utils.DummyLM([]), use_native_function_calling=True) + + result = await adapter.acall(NativeToolLM(), {}, ToolSignature, [], {"tools": [dspy.Tool(add)]}) + + assert result[0]["tool_calls"].tool_calls[0].id == "call_add" + + def test_two_step_adapter_parse(): class ComplexSignature(dspy.Signature): input_text: str = dspy.InputField() diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py index 315aa5b1c7..6cdbf4363f 100644 --- a/tests/streaming/test_streaming.py +++ b/tests/streaming/test_streaming.py @@ -1059,16 +1059,15 @@ def is_streamable(cls) -> bool: return True @classmethod - def adapt_to_native_lm_feature(cls, signature, field_name, lm, lm_kwargs): - return signature.delete(field_name) + def default_parse_output(cls, ctx): + return CustomType(message=ctx.adapter.lm_parts_to_text(ctx.parts)) @classmethod def parse_stream_chunk(cls, chunk): - return CustomType(message=chunk.choices[0].delta.content) - - @classmethod - def parse_lm_response(cls, response: dict) -> "CustomType": - return CustomType(message=response.split("\n\n")[0]) + content = chunk.choices[0].delta.content + if content in {"Hello", "World"}: + return CustomType(message=content) + return None class CustomSignature(dspy.Signature): question: str = dspy.InputField() @@ -1082,6 +1081,9 @@ class CustomSignature(dspy.Signature): ) async def stream(*args, **kwargs): + yield ModelResponseStream( + model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ## answer ## ]]\n\n"))] + ) yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="Hello"))]) yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="World"))]) yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="\n\n"))]) @@ -1091,9 +1093,7 @@ async def stream(*args, **kwargs): yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))]) with mock.patch("litellm.acompletion", side_effect=stream): - with dspy.context( - lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter(native_response_types=[CustomType]) - ): + with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter()): output = program(question="why did a chicken cross the kitchen?") all_chunks = [] async for value in output: @@ -1104,6 +1104,7 @@ async def stream(*args, **kwargs): assert value.answer.message == "HelloWorld" assert all(isinstance(chunk.chunk, CustomType) for chunk in all_chunks) + assert [chunk.chunk.message for chunk in all_chunks] == ["Hello", "World"] @pytest.mark.anyio From f77af4deeb03eabc42ab92c5d40cd9c506744d76 Mon Sep 17 00:00:00 2001 From: isaacbmiller Date: Sun, 24 May 2026 00:32:56 -0400 Subject: [PATCH 22/23] feat(predict): add ReActV2 runtime --- dspy/predict/__init__.py | 2 + dspy/predict/reactv2.py | 258 ++++++++++++++++++++++++++++++++++ tests/predict/test_reactv2.py | 179 +++++++++++++++++++++++ 3 files changed, 439 insertions(+) create mode 100644 dspy/predict/reactv2.py create mode 100644 tests/predict/test_reactv2.py diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py index 906ef90ae9..e3ad562a6a 100644 --- a/dspy/predict/__init__.py +++ b/dspy/predict/__init__.py @@ -8,6 +8,7 @@ from dspy.predict.predict import Predict from dspy.predict.program_of_thought import ProgramOfThought from dspy.predict.react import ReAct, Tool +from dspy.predict.reactv2 import ReActV2 from dspy.predict.refine import Refine from dspy.predict.rlm import RLM @@ -21,6 +22,7 @@ "Predict", "ProgramOfThought", "ReAct", + "ReActV2", "Refine", "RLM", "Tool", diff --git a/dspy/predict/reactv2.py b/dspy/predict/reactv2.py new file mode 100644 index 0000000000..55fddfb311 --- /dev/null +++ b/dspy/predict/reactv2.py @@ -0,0 +1,258 @@ +import logging +import traceback +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable + +import dspy +from dspy.adapters.types.history import History +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls +from dspy.primitives.module import Module +from dspy.signatures.signature import ensure_signature +from dspy.utils.exceptions import AdapterParseError, ContextWindowExceededError + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from dspy.signatures.signature import Signature + +_ANNOTATION_TO_JSON_TYPE = { + str: "string", + int: "integer", + float: "number", + bool: "boolean", + list: "array", +} +_RECOVERABLE_FORCED_SUBMIT_ERRORS = (AdapterParseError, ContextWindowExceededError, ValueError) + + +@dataclass(frozen=True) +class ToolExecutionResult: + value: object + is_error: bool = False + + +def _build_submit_tool(signature: type["Signature"]) -> Tool: + outputs = ", ".join([f"`{k}`" for k in signature.output_fields]) + output_args = {} + output_arg_types = {} + for name, field in signature.output_fields.items(): + annotation = getattr(field, "annotation", str) + # TODO(reactv2): Derive full JSON schema for complex output annotations instead of this primitive map. + output_args[name] = {"type": _ANNOTATION_TO_JSON_TYPE.get(annotation, "string")} + output_arg_types[name] = annotation + + return Tool( + func=lambda **kwargs: kwargs, + name="submit", + desc=f"Call this tool to end the task and return your final answer. Takes: {outputs}.", + args=output_args, + arg_types=output_arg_types, + ) + + +def _optional_input_fields(signature: type["Signature"]) -> dict[str, object]: + fields = deepcopy(signature.input_fields) + for field in fields.values(): + field.default = None + return fields + + +class ReActV2(Module): + def __init__(self, signature: type["Signature"] | str, tools: list[Callable], max_iters: int = 20): + super().__init__() + self.signature = signature = ensure_signature(signature) + self.max_iters = max_iters + + tools = [tool if isinstance(tool, Tool) else Tool(tool) for tool in tools] + self.tools = {tool.name: tool for tool in tools} + self.tools["submit"] = _build_submit_tool(signature) + + react_signature = ( + dspy.Signature(_optional_input_fields(signature), self._build_instructions()) + .append("history", dspy.InputField(), type_=dspy.History) + .append("tools", dspy.InputField(), type_=list[dspy.Tool]) + .append( + "tool_call_results", + dspy.InputField(default=None), + type_=dspy.ToolCallResults | None, + ) + .append("next_thought", dspy.OutputField(), type_=dspy.Reasoning) + .append("tool_calls", dspy.OutputField(), type_=dspy.ToolCalls) + ) + extract_signature = dspy.Signature( + {**signature.input_fields, **signature.output_fields}, + signature.instructions, + ).append( + "trajectory", + dspy.InputField(desc="The agent's history of thoughts, actions, and tool call results"), + type_=str, + ) + self.react = dspy.Predict(react_signature) + self.extract = dspy.ChainOfThought(extract_signature) + + def _build_instructions(self) -> str: + inputs = ", ".join([f"`{k}`" for k in self.signature.input_fields]) + outputs = ", ".join([f"`{k}`" for k in self.signature.output_fields]) + instructions = [f"{self.signature.instructions}\n"] if self.signature.instructions else [] + instructions.extend( + [ + f"You are an Agent. Given {inputs}, use tools to produce {outputs}.", + "Each turn: think, then call one or more tools. After each tool call you receive a tool result.", + "When you have enough information to answer, call `submit` to finish.", + "\nAvailable tools:\n", + ] + ) + instructions.extend(f"({idx + 1}) {tool}" for idx, tool in enumerate(self.tools.values())) + return "\n".join(instructions) + + def forward(self, **input_args): + history = input_args.pop("history", dspy.History(messages=[])) + max_iters = input_args.pop("max_iters", self.max_iters) + tool_list = list(self.tools.values()) + pending_inputs = dict(input_args) + + break_reason = None + for idx in range(max_iters): + try: + pred = self.react(history=history, tools=tool_list, **pending_inputs) + except ContextWindowExceededError: + history.compact_if_needed() + try: + pred = self.react(history=history, tools=tool_list, **pending_inputs) + except ContextWindowExceededError: + logger.warning("Context window exceeded after compaction, ending loop.") + break_reason = "context_overflow" + break + except (AdapterParseError, ValueError) as err: + logger.warning(f"Agent iteration {idx} failed: {_fmt_exc(err)}") + break_reason = "parse_error" + break + + if pred.tool_calls is None or not pred.tool_calls.tool_calls: + logger.warning("Agent returned no tool calls, ending loop.") + break_reason = "no_tool_calls" + break + + tool_calls = pred.tool_calls.with_call_ids(f"call_{len(history.messages)}") + tool_results = [self._execute_tool_call(tool_call) for tool_call in tool_calls.tool_calls] + self._append_tool_turn( + history, + inputs=pending_inputs, + next_thought=pred.next_thought, + tool_calls=tool_calls, + tool_results=tool_results, + ) + pending_inputs = {} + + for tool_call, tool_result in zip(tool_calls.tool_calls, tool_results, strict=True): + if tool_call.name == "submit" and not tool_result.is_error: + history.messages[-1].update(tool_result.value) + return dspy.Prediction(history=history, termination_reason="submit", **tool_result.value) + + return self._forced_submit(history, pending_inputs, break_reason=break_reason) + + def _forced_submit(self, history: History, input_args: dict[str, object], break_reason: str | None = None): + tool_list = list(self.tools.values()) + adapter = dspy.settings.adapter or dspy.ChatAdapter() + call_config = adapter.force_tool_call_config("submit") + + try: + pred = self.react(history=history, tools=tool_list, config=call_config, **input_args) + except _RECOVERABLE_FORCED_SUBMIT_ERRORS as err: + logger.debug(f"Forced submit tier 1 (react) failed: {_fmt_exc(err)}") + pred = None + + if pred and pred.tool_calls and pred.tool_calls.tool_calls: + for tool_call in pred.tool_calls.tool_calls: + if tool_call.name != "submit": + continue + try: + result = self.tools["submit"](**tool_call.args) + except ValueError as err: + logger.debug(f"Forced submit tool execution failed: {_fmt_exc(err)}") + continue + + tool_calls = ToolCalls(tool_calls=[tool_call]).with_call_ids(f"call_{len(history.messages)}") + self._append_tool_turn( + history, + inputs=input_args, + next_thought=pred.next_thought, + tool_calls=tool_calls, + tool_results=[ToolExecutionResult(value=result, is_error=False)], + ) + try: + history.messages[-1].update(result) + return dspy.Prediction(history=history, termination_reason="forced_submit", **result) + except TypeError as err: + logger.debug(f"Forced submit result was not a valid output mapping: {_fmt_exc(err)}") + + try: + trajectory_text = self._render_history_as_text(history) + extract = self.extract(trajectory=trajectory_text, **input_args) + result = {k: getattr(extract, k) for k in self.signature.output_fields if hasattr(extract, k)} + if any(value is not None for value in result.values()): + history.append(result) + return dspy.Prediction(history=history, termination_reason="extract", **result) + except _RECOVERABLE_FORCED_SUBMIT_ERRORS as err: + logger.debug(f"Forced submit tier 2 (extract) failed: {_fmt_exc(err)}") + + return dspy.Prediction(history=history, termination_reason=break_reason or "failed") + + def _execute_tool_call(self, tool_call: ToolCalls.ToolCall) -> ToolExecutionResult: + tool = self.tools.get(tool_call.name) + if tool is None: + return ToolExecutionResult(value=f"Unknown tool: {tool_call.name}", is_error=True) + try: + return ToolExecutionResult(value=tool(**tool_call.args), is_error=False) + except Exception as err: + return ToolExecutionResult(value=f"Execution error in {tool_call.name}: {_fmt_exc(err)}", is_error=True) + + @staticmethod + def _append_tool_turn( + history: History, + *, + inputs: dict[str, object] | None = None, + next_thought, + tool_calls: ToolCalls, + tool_results: list[ToolExecutionResult], + ) -> None: + history.append( + { + **(inputs or {}), + "next_thought": next_thought, + "tool_calls": tool_calls, + "tool_call_results": ToolCallResults.from_tool_calls_and_values( + tool_calls.tool_calls, + [tool_result.value for tool_result in tool_results], + [tool_result.is_error for tool_result in tool_results], + ), + } + ) + + def _render_history_as_text(self, history: History) -> str: + lines = [] + for event in history.messages: + thought = event.get("next_thought") + tool_calls = event.get("tool_calls") + tool_call_results = event.get("tool_call_results") + if thought: + lines.append(f"[Thought] {thought}") + if isinstance(tool_calls, ToolCalls): + for tool_call in tool_calls.tool_calls: + args = ", ".join(f"{key}={value!r}" for key, value in (tool_call.args or {}).items()) + lines.append(f"[Action] {tool_call.name}({args})") + if isinstance(tool_call_results, ToolCallResults): + for result in tool_call_results.tool_call_results: + prefix = "[Error]" if result.is_error else "[Observation]" + lines.append(f"{prefix} {result.name}: {result.value}") + for key, value in event.items(): + if key in {"next_thought", "tool_calls", "tool_call_results"}: + continue + label = "[Input]" if key in self.signature.input_fields else "[Output]" + lines.append(f"{label} {key}: {value}") + return "\n".join(lines) + + +def _fmt_exc(err: BaseException, *, limit: int = 5) -> str: + return "\n" + "".join(traceback.format_exception(type(err), err, err.__traceback__, limit=limit)).strip() diff --git a/tests/predict/test_reactv2.py b/tests/predict/test_reactv2.py new file mode 100644 index 0000000000..f074b3d57c --- /dev/null +++ b/tests/predict/test_reactv2.py @@ -0,0 +1,179 @@ +import dspy +from dspy.predict.reactv2 import ReActV2, ToolExecutionResult, _build_submit_tool +from dspy.utils.dummies import DummyLM + + +def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + +def test_submit_tool_returns_dict(): + signature = dspy.Signature("question -> answer") + submit = _build_submit_tool(signature) + + assert submit(answer="42") == {"answer": "42"} + + +def test_basic_forward_with_submit_records_history_messages(): + lm = DummyLM( + [ + {"next_thought": "I should add.", "tool_calls": [{"name": "add", "args": {"a": 1, "b": 2}}]}, + {"next_thought": "I have the answer.", "tool_calls": [{"name": "submit", "args": {"answer": "3"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="What is 1+2?") + + assert result.answer == "3" + assert len(result.history.messages) == 2 + assert result.history.messages[0]["question"] == "What is 1+2?" + assert result.history.messages[0]["next_thought"] == "I should add." + assert result.history.messages[0]["tool_call_results"].tool_call_results[0].value == 3 + assert result.history.messages[-1]["answer"] == "3" + second_call_user_messages = [message["content"] for message in lm.history[1]["messages"] if message["role"] == "user"] + assert any("tool_call_results" in content for content in second_call_user_messages) + assert all("None" not in content for content in second_call_user_messages) + assert "What is 1+2?" in second_call_user_messages[0] + assert "What is 1+2?" not in second_call_user_messages[-1] + + +def test_forward_with_native_tool_calling_renders_tool_results_as_tool_messages(): + class NativeToolLoopLM(dspy.BaseLM): + def __init__(self): + super().__init__(model="openai/gpt-5-nano", cache=False) + self.calls = [] + + @property + def supported_params(self): + return frozenset() + + @property + def supports_function_calling(self): + return True + + @property + def supports_reasoning(self): + return False + + @property + def supports_response_schema(self): + return False + + def __call__(self, messages, **kwargs): + self.calls.append({"messages": messages, "kwargs": kwargs}) + if len(self.calls) == 1: + return [ + { + "text": "[[ ## next_thought ## ]]\nI should add.\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + "id": "call_add", + "type": "function", + } + ], + } + ] + return [ + { + "text": "[[ ## next_thought ## ]]\nI can answer.\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "function": {"name": "submit", "arguments": '{"answer": "3"}'}, + "id": "call_submit", + "type": "function", + } + ], + } + ] + + lm = NativeToolLoopLM() + with dspy.context(lm=lm, adapter=dspy.ChatAdapter(use_native_function_calling=True)): + result = ReActV2("question -> answer", tools=[add])(question="What is 1+2?") + + assert result.answer == "3" + assert result.history.messages[0]["tool_call_results"].tool_call_results[0].call_id == "call_add" + assert any(message["role"] == "tool" and message["tool_call_id"] == "call_add" for message in lm.calls[1]["messages"]) + assert lm.calls[0]["kwargs"]["tools"][0]["function"]["name"] == "add" + + +def test_react_signature_keeps_optional_tool_call_results_input(): + react = ReActV2("question -> answer", tools=[add]) + + assert react.react.signature.input_fields["question"].default is None + field = react.react.signature.input_fields["tool_call_results"] + assert field.default is None + assert field.annotation == dspy.ToolCallResults | None + + +def test_forward_with_existing_history_does_not_append_empty_input_event(): + lm = DummyLM( + [ + {"next_thought": "I have the answer.", "tool_calls": [{"name": "submit", "args": {"answer": "3"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + history = dspy.History(messages=[{"question": "What is 1+2?"}]) + + result = react(history=history) + + assert result.answer == "3" + assert {} not in result.history.messages + assert result.history.messages[0] == {"question": "What is 1+2?"} + + +def test_unknown_tool_returns_error_observation(): + lm = DummyLM( + [ + {"next_thought": "Call fake.", "tool_calls": [{"name": "nonexistent", "args": {}}]}, + {"next_thought": "Now submit.", "tool_calls": [{"name": "submit", "args": {"answer": "ok"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="test") + + assert result.answer == "ok" + tool_results = [ + result + for message in result.history.messages + for result in getattr(message.get("tool_call_results"), "tool_call_results", []) + ] + assert any(result.is_error and "Unknown tool" in str(result.value) for result in tool_results) + + +def test_append_tool_turn_records_observation_ids(): + history = dspy.History(messages=[]) + tool_calls = dspy.ToolCalls.from_dict_list([{"name": "add", "args": {"a": 1, "b": 2}, "id": "call_add"}]) + + ReActV2._append_tool_turn( + history, + next_thought="add", + tool_calls=tool_calls, + tool_results=[ToolExecutionResult(value=3)], + ) + + tool_result = history.messages[0]["tool_call_results"].tool_call_results[0] + assert tool_result.call_id == "call_add" + assert tool_result.name == "add" + + +def test_forced_submit_runs_when_loop_returns_no_tool_calls(): + lm = DummyLM( + [ + {"next_thought": "No call.", "tool_calls": []}, + {"next_thought": "Force submit.", "tool_calls": [{"name": "submit", "args": {"answer": "done"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="test") + + assert result.answer == "done" + assert result.termination_reason == "forced_submit" From 28a56068a6a7975f446dcb8018a21f8d5377693d Mon Sep 17 00:00:00 2001 From: Maxime Rivest Date: Sun, 24 May 2026 07:58:11 -0400 Subject: [PATCH 23/23] Move LM capabilities into BaseLM --- dspy/__init__.py | 18 +++- dspy/adapters/_type_runtime.py | 13 +-- dspy/adapters/base.py | 32 ++++--- dspy/clients/__init__.py | 7 +- dspy/clients/base_lm.py | 88 +++++++++++++++++--- dspy/clients/lm.py | 12 ++- dspy/predict/predict.py | 57 ++++++++++++- dspy/utils/exceptions.py | 147 +++++++++++++++++++++++++++++---- 8 files changed, 314 insertions(+), 60 deletions(-) diff --git a/dspy/__init__.py b/dspy/__init__.py index cc9d1fe5e8..56fc761cf6 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -8,7 +8,23 @@ from dspy.clients import * # isort: skip from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, ToolCallResults, Code, Reasoning # isort: skip from dspy.primitives.sandbox_serializable import SandboxSerializable # isort: skip -from dspy.utils.exceptions import ContextWindowExceededError +from dspy.utils.exceptions import ( + ContextWindowExceededError, + DSPyError, + LMAuthError, + LMBillingError, + LMConfigurationError, + LMError, + LMInvalidRequestError, + LMNotConfiguredError, + LMProviderError, + LMRateLimitError, + LMServerError, + LMTimeoutError, + LMTransportError, + LMUnsupportedFeatureError, + LMUnsupportedModelError, +) from dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_logging from dspy.utils.asyncify import asyncify from dspy.utils.syncify import syncify diff --git a/dspy/adapters/_type_runtime.py b/dspy/adapters/_type_runtime.py index 757aa3ae22..9c5e5d1985 100644 --- a/dspy/adapters/_type_runtime.py +++ b/dspy/adapters/_type_runtime.py @@ -10,26 +10,17 @@ if TYPE_CHECKING: from dspy.adapters.base import Adapter + from dspy.clients.base_lm import BaseLM from dspy.signatures.signature import Signature -@dataclass(frozen=True) -class _LMCapabilities: - model: str - model_type: str - supported_params: frozenset[str] - supports_function_calling: bool - supports_response_schema: bool - supports_reasoning: bool - - @dataclass(frozen=True) class _CallContext: adapter: Adapter use_native_function_calling: bool allow_parallel_tool_calls: bool | None native_response_types: tuple[type[object], ...] - lm: _LMCapabilities + lm: BaseLM lm_kwargs: dict[str, object] = field(default_factory=dict) lm_default_kwargs: dict[str, object] = field(default_factory=dict) diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index 599067981d..86d3eeae01 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -19,7 +19,6 @@ from dspy.adapters._type_runtime import ( _AdapterCallPlan, _CallContext, - _LMCapabilities, _merge_lm_config, _OutputParser, _TypeFeatureHandler, @@ -30,7 +29,7 @@ from dspy.adapters.types.reasoning import Reasoning from dspy.adapters.types.tool import ToolCallResults, ToolCalls from dspy.adapters.utils import format_field_value, parse_value -from dspy.clients.base_lm import BaseLM +from dspy.clients.base_lm import BaseLM, LMCapabilities from dspy.clients.openai_format import ( lm_response_from_legacy_outputs, message_to_openai_chat, @@ -59,6 +58,17 @@ def signature(self) -> type[Signature]: return self.call_plan.render_signature +class _AdapterContextLM(BaseLM): + """Minimal BaseLM used when rendering adapter messages without a real LM.""" + + def __init__(self, *, use_native_tool_calls: bool = False): + super().__init__(model="", model_type="chat", temperature=None, max_tokens=None) + self._capabilities = LMCapabilities(function_calling=use_native_tool_calls) + + def get_capabilities(self) -> LMCapabilities: + return self._capabilities + + class Adapter: """Base Adapter class. @@ -116,14 +126,7 @@ def build_call_context(self, lm: BaseLM, lm_kwargs: dict[str, Any] | None = None use_native_function_calling=self.use_native_function_calling, allow_parallel_tool_calls=self.allow_parallel_tool_calls, native_response_types=tuple(self.native_response_types), - lm=_LMCapabilities( - model=lm.model, - model_type=getattr(lm, "model_type", "chat"), - supported_params=frozenset(getattr(lm, "supported_params", set())), - supports_function_calling=bool(getattr(lm, "supports_function_calling", False)), - supports_response_schema=bool(getattr(lm, "supports_response_schema", False)), - supports_reasoning=bool(getattr(lm, "supports_reasoning", False)), - ), + lm=lm, lm_kwargs=dict(lm_kwargs or {}), lm_default_kwargs=dict(getattr(lm, "kwargs", {}) or {}), ) @@ -134,14 +137,7 @@ def _default_call_context(self, *, use_native_tool_calls: bool = False) -> _Call use_native_function_calling=use_native_tool_calls, allow_parallel_tool_calls=self.allow_parallel_tool_calls, native_response_types=tuple(self.native_response_types), - lm=_LMCapabilities( - model="", - model_type="chat", - supported_params=frozenset(), - supports_function_calling=use_native_tool_calls, - supports_response_schema=False, - supports_reasoning=False, - ), + lm=_AdapterContextLM(use_native_tool_calls=use_native_tool_calls), ) def value_to_lm_parts(self, value: object, field_info: FieldInfo) -> list[LMPart]: diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py index c371f3e06b..be5240e20f 100644 --- a/dspy/clients/__init__.py +++ b/dspy/clients/__init__.py @@ -4,7 +4,7 @@ from typing import Any from dspy.clients._litellm import get_litellm -from dspy.clients.base_lm import BaseLM, inspect_history +from dspy.clients.base_lm import BaseLM, LMCapabilities, inspect_history from dspy.clients.cache import Cache from dspy.clients.embedding import Embedder from dspy.clients.lm import LM @@ -39,7 +39,7 @@ def configure_cache( safe_types: Additional types to allow when restrict_pickle is True. """ - DSPY_CACHE = Cache( + dspy_cache = Cache( enable_disk_cache, enable_memory_cache, disk_cache_dir, @@ -52,7 +52,7 @@ def configure_cache( import dspy # Update the reference to point to the new cache - dspy.cache = DSPY_CACHE + dspy.cache = dspy_cache @@ -112,6 +112,7 @@ def disable_litellm_logging(): __all__ = [ "BaseLM", + "LMCapabilities", "LM", "Provider", "TrainingJob", diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.py index 760a2679b5..2acb49e3b7 100644 --- a/dspy/clients/base_lm.py +++ b/dspy/clients/base_lm.py @@ -1,7 +1,11 @@ +import copy as copy_module import datetime import uuid +from dataclasses import dataclass, field from typing import Any, TextIO +from typing_extensions import Self + from dspy.dsp.utils import settings from dspy.utils.callback import with_callbacks from dspy.utils.inspect_history import pretty_print_history @@ -10,6 +14,27 @@ GLOBAL_HISTORY = [] +@dataclass(frozen=True) +class LMCapabilities: + """Optional model and deployment metadata for an LM backend. + + Capabilities are descriptive hints. Adapters can use them to select native + paths, but concrete LM implementations still decide how to handle requests. + """ + + function_calling: bool = False + reasoning: bool = False + response_schema: bool = False + streaming: bool = False + input_image: bool = False + input_audio: bool = False + input_file: bool = False + output_image: bool = False + output_audio: bool = False + tool_results: bool = False + extensions: dict[str, Any] = field(default_factory=dict) + + class BaseLM: """Base class for handling LLM calls. @@ -60,32 +85,55 @@ def forward(self, prompt, messages=None, **kwargs): ``` """ - def __init__(self, model, model_type="chat", temperature=0.0, max_tokens=1000, cache=True, **kwargs): + def __init__( + self, + model, + model_type="chat", + temperature=0.0, + max_tokens=1000, + cache=True, + callbacks=None, + num_retries=0, + **kwargs, + ): self.model = model self.model_type = model_type self.cache = cache + self.callbacks = callbacks or [] + self.num_retries = num_retries self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs) self.history = [] + self._warned_zero_temp_rollout = False + + @property + def capabilities(self) -> LMCapabilities: + """Native metadata available for this model instance.""" + return self.get_capabilities() + + def get_capabilities(self) -> LMCapabilities: + """Return optional native model and deployment hints.""" + return LMCapabilities() @property def supports_function_calling(self) -> bool: """Whether the model supports function calling (tool use).""" - return False + return self.capabilities.function_calling @property def supports_reasoning(self) -> bool: """Whether the model supports native reasoning (extended thinking).""" - return False + return self.capabilities.reasoning @property def supports_response_schema(self) -> bool: """Whether the model supports structured output via response schema.""" - return False + return self.capabilities.response_schema @property def supported_params(self) -> set[str]: """Set of supported OpenAI-style parameter names for the model.""" - return set() + supported = self.capabilities.extensions.get("supported_params", set()) + return set(supported) if supported else set() def _process_lm_response(self, response, prompt, messages, **kwargs): merged_kwargs = {**self.kwargs, **kwargs} @@ -190,23 +238,43 @@ async def aforward( """ raise NotImplementedError("Subclasses must implement this method.") + def dump_state(self) -> dict[str, Any]: + """Return a sanitized reconstruction state for this LM.""" + filtered_kwargs = {key: value for key, value in self.kwargs.items() if key != "api_key"} + return { + "model": self.model, + "model_type": self.model_type, + "cache": self.cache, + "num_retries": self.num_retries, + **filtered_kwargs, + } + + @classmethod + def load_state(cls, state: dict[str, Any]) -> Self: + """Reconstruct this LM from `dump_state()` output.""" + return cls(**state) + def copy(self, **kwargs): """Returns a copy of the language model with possibly updated parameters. Any provided keyword arguments update the corresponding attributes or LM kwargs of the copy. For example, ``lm.copy(rollout_id=1, temperature=1.0)`` returns an LM whose requests use a different rollout ID at non-zero temperature to bypass cache collisions. - """ - import copy + The default implementation uses a shallow runtime copy so provider clients, sessions, + and local model handles are preserved by reference while DSPy-owned mutable state is + isolated on the copy. + """ - new_instance = copy.deepcopy(self) + new_instance = copy_module.copy(self) new_instance.history = [] + new_instance.callbacks = list(getattr(self, "callbacks", []) or []) + new_instance.kwargs = dict(getattr(self, "kwargs", {}) or {}) for key, value in kwargs.items(): - if hasattr(self, key): + if hasattr(new_instance, key): setattr(new_instance, key, value) - if (key in self.kwargs) or (not hasattr(self, key)): + if (key in new_instance.kwargs) or (not hasattr(self, key)): if value is None: new_instance.kwargs.pop(key, None) else: diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py index 51ad677e0e..1405a745c9 100644 --- a/dspy/clients/lm.py +++ b/dspy/clients/lm.py @@ -20,7 +20,7 @@ from dspy.utils.callback import BaseCallback from dspy.utils.exceptions import ContextWindowExceededError -from .base_lm import BaseLM +from .base_lm import BaseLM, LMCapabilities logger = logging.getLogger(__name__) @@ -123,6 +123,16 @@ def _provider_name(self) -> str: return self.model.split("/", 1)[0] return "openai" + def get_capabilities(self) -> LMCapabilities: + params = self.supported_params + return LMCapabilities( + function_calling=self.supports_function_calling, + reasoning=self.supports_reasoning, + response_schema=self.supports_response_schema, + streaming=True, + extensions={"supported_params": params}, + ) + @property def supports_function_calling(self) -> bool: return _get_litellm().supports_function_calling(model=self.model) diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py index 8b10493156..6212806bf7 100644 --- a/dspy/predict/predict.py +++ b/dspy/predict/predict.py @@ -1,3 +1,4 @@ +import importlib import logging import random import types @@ -21,6 +22,7 @@ logger = logging.getLogger(__name__) UNSAFE_LM_STATE_KEYS = {"api_base", "base_url", "model_list"} +_LM_CLASS_STATE_KEY = "_dspy_lm_class" def _sanitize_lm_state(lm_state: dict, allow_unsafe_lm_state: bool) -> dict: @@ -99,7 +101,7 @@ def dump_state(self, json_mode=True): state["demos"].append(demo.toDict()) state["signature"] = self.signature.dump_state() - state["lm"] = self.lm.dump_state() if self.lm else None + state["lm"] = _dump_lm_state(self.lm) if self.lm else None return state def load_state(self, state: dict, *, allow_unsafe_lm_state: bool = False) -> "Predict": @@ -121,7 +123,7 @@ def load_state(self, state: dict, *, allow_unsafe_lm_state: bool = False) -> "Pr self.signature = self.signature.load_state(state["signature"]) sanitized_lm_state = _sanitize_lm_state(state["lm"], allow_unsafe_lm_state) if state["lm"] else None - self.lm = LM(**sanitized_lm_state) if sanitized_lm_state else None + self.lm = _load_lm_from_state(sanitized_lm_state) if sanitized_lm_state else None if "extended_signature" in state: # legacy, up to and including 2.5, for CoT. raise NotImplementedError("Loading extended_signature is no longer supported in DSPy 2.6+") @@ -291,6 +293,57 @@ def get_config(self): def __repr__(self): return f"{self.__class__.__name__}({self.signature})" + +def _dump_lm_state(lm: BaseLM) -> dict[str, Any]: + """Return LM state with enough type information to reconstruct it.""" + lm_state = dict(lm.dump_state()) + lm_state.setdefault(_LM_CLASS_STATE_KEY, f"{type(lm).__module__}.{type(lm).__qualname__}") + return lm_state + + +def _load_lm_from_state(lm_state: dict[str, Any]) -> BaseLM: + """Reconstruct a legacy or custom BaseLM from serialized state. + + Older saved programs did not record the concrete LM class, so they are + loaded through the legacy LiteLLM-backed `dspy.clients.lm.LM` constructor. + Newer states record the concrete class so `Predict` can round-trip any + importable `BaseLM` subclass. + """ + lm_state = dict(lm_state) + class_path = lm_state.pop(_LM_CLASS_STATE_KEY, None) + if class_path is None: + return LM(**lm_state) + + lm_cls = _import_lm_class(class_path) + if not issubclass(lm_cls, BaseLM): + raise TypeError(f"Serialized LM class `{class_path}` must be a subclass of dspy.BaseLM.") + + load_state = getattr(lm_cls, "load_state", None) + if callable(load_state): + return load_state(lm_state) + return lm_cls(**lm_state) + + +def _import_lm_class(class_path: str) -> type: + try: + module_name, class_name = class_path.rsplit(".", 1) + except ValueError as exc: + raise ValueError(f"Invalid serialized LM class path: {class_path!r}") from exc + + try: + module = importlib.import_module(module_name) + lm_cls = getattr(module, class_name) + except (ImportError, AttributeError) as exc: + raise ImportError( + f"Could not import serialized LM class `{class_path}`. Ensure the class is importable, " + "or load an older state without the concrete LM class marker." + ) from exc + + if not isinstance(lm_cls, type): + raise TypeError(f"Serialized LM class `{class_path}` did not resolve to a class.") + return lm_cls + + def _get_type_name(type_annotation) -> str: """Helper method to get the name for a type annotation.""" diff --git a/dspy/utils/exceptions.py b/dspy/utils/exceptions.py index 928bbe351b..dbfd79ae1c 100644 --- a/dspy/utils/exceptions.py +++ b/dspy/utils/exceptions.py @@ -1,25 +1,144 @@ +from __future__ import annotations from dspy.signatures.signature import Signature -class ContextWindowExceededError(Exception): - """Raised when the prompt exceeds the model's context window. +class DSPyError(Exception): + """Base class for DSPy errors with structured metadata.""" - Any `BaseLM` subclass should raise this error (or a subclass of it) when the - request fails because the input is too long for the model. Adapters and some - modules rely on catching this specific type to decide whether a fallback - retry is appropriate. + default_code: str | None = None - Args: - model: The model identifier that rejected the request. - message: Description of the error. Defaults to `"Context window exceeded"`. - """ - - def __init__(self, *, model: str | None = None, message: str = "Context window exceeded"): + def __init__( + self, + message: str = "", + *, + code: str | None = None, + model: str | None = None, + provider: str | None = None, + provider_code: str | None = None, + status: int | None = None, + request_id: str | None = None, + retry_after: float | None = None, + ): + self.message = message + self.code = code or self.default_code self.model = model - msg = message + self.provider = provider + self.provider_code = provider_code + self.status = status + self.request_id = request_id + self.retry_after = retry_after + prefix = f"[{model}] " if model else "" - super().__init__(f"{prefix}{msg}") + super().__init__(f"{prefix}{message}" if message else prefix.rstrip()) + + +class LMError(DSPyError): + """Base class for language model errors.""" + + default_code = "lm_error" + + +class LMTransportError(LMError): + """The LM request failed before the provider returned a response.""" + + default_code = "transport" + + +class LMConfigurationError(LMError): + """The LM or provider client is not configured correctly.""" + + default_code = "configuration" + + +class LMNotConfiguredError(LMConfigurationError): + """The LM is missing required provider configuration or credentials.""" + + default_code = "not_configured" + + +class LMUnsupportedFeatureError(LMError): + """The LM does not support a requested feature.""" + + default_code = "unsupported_feature" + + def __init__( + self, + message: str = "", + *, + features: list[str] | None = None, + issues: list[str] | None = None, + **kwargs, + ): + self.features = list(features or []) + self.issues = list(issues or []) + super().__init__(message, **kwargs) + + +class LMProviderError(LMError): + """The provider returned an error response.""" + + default_code = "provider" + + +class LMAuthError(LMProviderError): + """The provider rejected the request because authentication failed.""" + + default_code = "auth" + + +class LMBillingError(LMProviderError): + """The provider rejected the request because billing or quota failed.""" + + default_code = "billing" + + +class LMRateLimitError(LMProviderError): + """The provider rate-limited the request.""" + + default_code = "rate_limit" + + +class LMInvalidRequestError(LMProviderError): + """The provider rejected the request shape or resource.""" + + default_code = "invalid_request" + + +class ContextWindowExceededError(LMInvalidRequestError): + """Raised when the prompt exceeds the model's context window.""" + + default_code = "context_window_exceeded" + + def __init__( + self, + *, + model: str | None = None, + message: str = "Context window exceeded", + **kwargs, + ): + super().__init__(message, model=model, **kwargs) + + +class LMUnsupportedModelError(LMInvalidRequestError): + """The requested model is unavailable or unsupported by the provider.""" + + default_code = "unsupported_model" + + +class LMTimeoutError(LMProviderError): + """The provider request timed out.""" + + default_code = "timeout" + + +class LMServerError(LMProviderError): + """The provider failed while handling the request.""" + + default_code = "server" + + +RETRYABLE_LM_ERRORS = (LMRateLimitError, LMTimeoutError, LMServerError, LMTransportError) class AdapterParseError(Exception):