diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index ba2e9d538c..c0add85326 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -21,7 +21,7 @@ jobs: with: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 + uses: zizmorcore/zizmor-action@a16621b09c6db4281f81a93cb393b05dcd7b7165 # v0.5.5 - name: Run actionlint uses: raven-actions/actionlint@205b530c5d9fa8f44ae9ed59f341a0db994aa6f8 # v2.1.2 diff --git a/README.md b/README.md index 4f36a4aa82..fe73d8cc89 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ To install the very latest from `main`: ```bash pip install git+https://github.com/stanfordnlp/dspy.git -```` +``` diff --git a/docs/docs/api/experimental/Citations.md b/docs/docs/api/experimental/Citations.md index 3738fd8922..a123680e99 100644 --- a/docs/docs/api/experimental/Citations.md +++ b/docs/docs/api/experimental/Citations.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/experimental/Document.md b/docs/docs/api/experimental/Document.md index 5b6fb6d643..3439461878 100644 --- a/docs/docs/api/experimental/Document.md +++ b/docs/docs/api/experimental/Document.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/Audio.md b/docs/docs/api/primitives/Audio.md index 129d9b358f..de097d709c 100644 --- a/docs/docs/api/primitives/Audio.md +++ b/docs/docs/api/primitives/Audio.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/Code.md b/docs/docs/api/primitives/Code.md index d84b4b05ad..9dacbd9486 100644 --- a/docs/docs/api/primitives/Code.md +++ b/docs/docs/api/primitives/Code.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/Image.md b/docs/docs/api/primitives/Image.md index fc46ce9cc7..924ad46ecb 100644 --- a/docs/docs/api/primitives/Image.md +++ b/docs/docs/api/primitives/Image.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/Tool.md b/docs/docs/api/primitives/Tool.md index 45f447e50c..3c58364743 100644 --- a/docs/docs/api/primitives/Tool.md +++ b/docs/docs/api/primitives/Tool.md @@ -7,7 +7,6 @@ members: - __call__ - acall - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/api/primitives/ToolCalls.md b/docs/docs/api/primitives/ToolCalls.md index 350bf0a9c8..5394ebb3ae 100644 --- a/docs/docs/api/primitives/ToolCalls.md +++ b/docs/docs/api/primitives/ToolCalls.md @@ -5,7 +5,6 @@ handler: python options: members: - - adapt_to_native_lm_feature - description - extract_custom_type_from_annotation - format diff --git a/docs/docs/faqs.md b/docs/docs/faqs.md index 3a5616fee9..65615faba1 100644 --- a/docs/docs/faqs.md +++ b/docs/docs/faqs.md @@ -19,7 +19,7 @@ The **DSPy** philosophy and abstraction differ significantly from other librarie ## Basic Usage -**How should I use DSPy for my task?** We wrote a [eight-step guide](learn/index.md) on this. In short, using DSPy is an iterative process. You first define your task and the metrics you want to maximize, and prepare a few example inputs — typically without labels (or only with labels for the final outputs, if your metric requires them). Then, you build your pipeline by selecting built-in layers (`modules`) to use, giving each layer a `signature` (input/output spec), and then calling your modules freely in your Python code. Lastly, you use a DSPy `optimizer` to compile your code into high-quality instructions, automatic few-shot examples, or updated LM weights for your LM. +**How should I use DSPy for my task?** We wrote an [eight-step guide](/learn/) on this. In short, using DSPy is an iterative process. You first define your task and the metrics you want to maximize, and prepare a few example inputs — typically without labels (or only with labels for the final outputs, if your metric requires them). Then, you build your pipeline by selecting built-in layers (`modules`) to use, giving each layer a `signature` (input/output spec), and then calling your modules freely in your Python code. Lastly, you use a DSPy `optimizer` to compile your code into high-quality instructions, automatic few-shot examples, or updated LM weights for your LM. **How do I convert my complex prompt into a DSPy pipeline?** See the same answer above. diff --git a/dspy/__init__.py b/dspy/__init__.py index 76afdb805d..56fc761cf6 100644 --- a/dspy/__init__.py +++ b/dspy/__init__.py @@ -6,9 +6,25 @@ from dspy.evaluate import Evaluate # isort: skip from dspy.clients import * # isort: skip -from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, Code, Reasoning # isort: skip +from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, ToolCallResults, Code, Reasoning # isort: skip from dspy.primitives.sandbox_serializable import SandboxSerializable # isort: skip -from dspy.utils.exceptions import ContextWindowExceededError +from dspy.utils.exceptions import ( + ContextWindowExceededError, + DSPyError, + LMAuthError, + LMBillingError, + LMConfigurationError, + LMError, + LMInvalidRequestError, + LMNotConfiguredError, + LMProviderError, + LMRateLimitError, + LMServerError, + LMTimeoutError, + LMTransportError, + LMUnsupportedFeatureError, + LMUnsupportedModelError, +) from dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_logging from dspy.utils.asyncify import asyncify from dspy.utils.syncify import syncify diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.py index c217d7260e..23a6631ebc 100644 --- a/dspy/adapters/__init__.py +++ b/dspy/adapters/__init__.py @@ -2,7 +2,18 @@ from dspy.adapters.chat_adapter import ChatAdapter from dspy.adapters.json_adapter import JSONAdapter from dspy.adapters.two_step_adapter import TwoStepAdapter -from dspy.adapters.types import Audio, Code, File, History, Image, Reasoning, Tool, ToolCalls, Type +from dspy.adapters.types import ( + Audio, + Code, + File, + History, + Image, + Reasoning, + Tool, + ToolCallResults, + ToolCalls, + Type, +) from dspy.adapters.xml_adapter import XMLAdapter __all__ = [ @@ -19,5 +30,6 @@ "TwoStepAdapter", "Tool", "ToolCalls", + "ToolCallResults", "Reasoning", ] diff --git a/dspy/adapters/_legacy_type_markers.py b/dspy/adapters/_legacy_type_markers.py new file mode 100644 index 0000000000..3a3a788ac3 --- /dev/null +++ b/dspy/adapters/_legacy_type_markers.py @@ -0,0 +1,145 @@ +"""Private compatibility helpers for legacy `dspy.Type` marker serialization.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +import json_repair + +from dspy.adapters.types.base_type import CUSTOM_TYPE_END_IDENTIFIER, CUSTOM_TYPE_START_IDENTIFIER +from dspy.core.types import LMAudioPart, LMBinaryPart, LMDocumentPart, LMImagePart, LMMessage, LMPart, LMTextPart + + +def _expand_legacy_custom_type_markers_in_chat_message(message: dict[str, Any]) -> dict[str, Any]: + """Expand legacy marker payloads in an OpenAI-chat-shaped user message.""" + if message.get("role") != "user" or not isinstance(message.get("content"), str): + return message + content = message["content"] + if CUSTOM_TYPE_START_IDENTIFIER not in content: + return message + return {**message, "content": _split_legacy_custom_type_text_to_blocks(content)} + + +def _expand_legacy_custom_type_markers_in_lm_message(message: LMMessage) -> LMMessage: + """Expand legacy marker payloads in user `LMMessage`s into normalized parts.""" + if message.role != "user": + return message + expanded_parts: list[LMPart] = [] + changed = False + for part in message.parts: + if not isinstance(part, LMTextPart) or CUSTOM_TYPE_START_IDENTIFIER not in part.text: + expanded_parts.append(part) + continue + changed = True + expanded_parts.extend(_split_legacy_custom_type_text_to_parts(part.text)) + return message.model_copy(update={"parts": expanded_parts}) if changed else message + + +def _split_legacy_custom_type_text_to_blocks(text: str) -> list[dict[str, Any]]: + blocks: list[dict[str, Any]] = [] + last_end = 0 + for match in re.finditer(_marker_pattern(), text, re.DOTALL): + start, end = match.span() + if start > last_end: + blocks.append({"type": "text", "text": text[last_end:start]}) + blocks.extend(_legacy_custom_type_payload_to_blocks(match.group(1).strip())) + last_end = end + if last_end < len(text): + blocks.append({"type": "text", "text": text[last_end:]}) + return blocks + + +def _split_legacy_custom_type_text_to_parts(text: str) -> list[LMPart]: + parts: list[LMPart] = [] + last_end = 0 + for match in re.finditer(_marker_pattern(), text, re.DOTALL): + start, end = match.span() + if start > last_end: + parts.append(LMTextPart(text=text[last_end:start])) + parts.extend(_legacy_custom_type_payload_to_parts(match.group(1).strip())) + last_end = end + if last_end < len(text): + parts.append(LMTextPart(text=text[last_end:])) + return parts + + +def _legacy_custom_type_payload_to_blocks(payload: str) -> list[dict[str, Any]]: + parsed = _parse_legacy_payload(payload) + if isinstance(parsed, list): + return [block if isinstance(block, dict) else {"type": "text", "text": str(block)} for block in parsed] + return [{"type": "text", "text": payload}] + + +def _legacy_custom_type_payload_to_parts(payload: str) -> list[LMPart]: + parsed = _parse_legacy_payload(payload) + if not isinstance(parsed, list): + return [LMTextPart(text=payload)] + return [_legacy_content_block_to_lm_part(block) for block in parsed] + + +def _parse_legacy_payload(payload: str) -> Any: + for parse_fn in (json.loads, _parse_doubly_quoted_json, json_repair.loads): + try: + return parse_fn(payload) + except Exception: + continue + return None + + +def _parse_doubly_quoted_json(value: str) -> Any: + # Legacy `Type` payloads can be JSON-encoded twice when the serialized + # marker string is nested inside a larger JSON value, e.g. list[Image]. + return json.loads(json.loads(f'"{value}"')) + + +def _legacy_content_block_to_lm_part(block: Any) -> LMPart: + if not isinstance(block, dict): + return LMTextPart(text=str(block)) + + block_type = block.get("type") + if block_type == "text": + return LMTextPart(text=block.get("text", "")) + if block_type == "image_url": + image_url = block.get("image_url", {}) + source = image_url.get("url") if isinstance(image_url, dict) else image_url + if isinstance(source, str) and source.startswith("data:") and "," in source: + media_type, data = _split_data_uri(source) + return LMImagePart(data=data, media_type=media_type) + return LMImagePart(url=source or "") + if block_type == "input_audio": + audio = block.get("input_audio", {}) + return LMAudioPart(data=audio.get("data", ""), media_type=f"audio/{audio.get('format', 'wav')}") + if block_type == "file": + file = block.get("file", {}) + if file.get("file_data") is not None: + media_type, data = _split_data_uri(file["file_data"]) + return LMBinaryPart(data=data, media_type=media_type, filename=file.get("filename"), metadata={"legacy_content_block": block}) + return LMBinaryPart(file_id=file.get("file_id", ""), filename=file.get("filename"), metadata={"legacy_content_block": block}) + if block_type == "document": + source = block.get("source", {}) + return LMDocumentPart( + source=source if isinstance(source, dict) else {"type": "text", "data": str(source)}, + citations=block.get("citations") or {"enabled": True}, + title=block.get("title"), + context=block.get("context"), + ) + + # Keep unknown provider-shaped blocks losslessly for the current BaseLM + # compatibility path. `openai_format.part_to_openai_blocks()` rehydrates + # this metadata before the legacy provider-shaped call. A later normalized + # type pass should replace this with an explicit opaque part or remove it + # when marker-based custom type rendering is retired. + return LMTextPart(text="", metadata={"legacy_content_block": block}) + + +def _marker_pattern() -> str: + return rf"{CUSTOM_TYPE_START_IDENTIFIER}(.*?){CUSTOM_TYPE_END_IDENTIFIER}" + + +def _split_data_uri(value: str) -> tuple[str, str]: + if value.startswith("data:") and "," in value: + header, data = value.split(",", 1) + return header.removeprefix("data:").split(";", 1)[0], data + return "application/octet-stream", value diff --git a/dspy/adapters/_type_feature_handlers.py b/dspy/adapters/_type_feature_handlers.py new file mode 100644 index 0000000000..097c4bb76a --- /dev/null +++ b/dspy/adapters/_type_feature_handlers.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, get_args + +from dspy.adapters._type_runtime import _AdapterCallPlan, _CallContext, _TypeFeatureHandler +from dspy.adapters.types.reasoning import Reasoning +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls +from dspy.core.types import ( + LMCitationPart, + LMConfig, + LMMessage, + LMOutput, + LMPart, + LMReasoningConfig, + LMThinkingPart, + LMToolChoice, +) +from dspy.experimental import Citations + + +@dataclass +class _RenderedTypeOutput: + parts: list[LMPart] = field(default_factory=list) + messages: list[LMMessage] = field(default_factory=list) + + @property + def consumed(self) -> bool: + return bool(self.parts or self.messages) + + +class _ToolTypeHandler(_TypeFeatureHandler): + def prepare(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + tool_call_output_field_name = _tool_call_output_field_name(call.source_signature) + tool_input_field_name = _tool_input_field_name(call.source_signature) + native = _native_tool_calling_enabled(call, ctx) + + if ctx.use_native_function_calling and tool_call_output_field_name is not None and tool_input_field_name is None: + raise ValueError( + f"You provided an output field {tool_call_output_field_name} to receive the tool calls information, " + "but did not provide any tools as the input. Please provide a list of tools as the input by adding an " + "input field with type `list[dspy.Tool]`." + ) + + if not native: + self._apply_text_parallel_policy(call, ctx) + return + + if ctx.allow_parallel_tool_calls is not None: + call.merge_config(LMConfig(tool_choice=LMToolChoice(parallel=ctx.allow_parallel_tool_calls))) + + if tool_input_field_name is not None: + value = call.inputs.get(tool_input_field_name) + if value is not None: + for tool in value if isinstance(value, list) else [value]: + tool = tool if isinstance(tool, Tool) else Tool.model_validate(tool) + call.tools.append(tool.to_lm_tool_spec()) + call.inputs.pop(tool_input_field_name, None) + call.delete_field(tool_input_field_name) + + if tool_call_output_field_name is not None: + call.delete_field(tool_call_output_field_name) + + for field_name, field_info in call.source_signature.input_fields.items(): + if not _annotation_includes(field_info.annotation, ToolCallResults): + continue + value = call.inputs.get(field_name) + if value is None: + continue + results = value if isinstance(value, ToolCallResults) else ToolCallResults.model_validate(value) + call.messages.extend(results.to_lm_messages()) + call.inputs.pop(field_name, None) + call.delete_field(field_name) + + def parse( + self, + values: dict[str, object], + output: LMOutput, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> None: + if not _native_tool_calling_enabled(call, ctx): + return + field_name = _tool_call_output_field_name(call.source_signature) + if field_name is None or field_name in values or not output.tool_calls: + return + values[field_name] = ToolCalls.from_dict_list( + [{"name": tool_call.name, "args": tool_call.args, "id": tool_call.id} for tool_call in output.tool_calls] + ) + + def format_output( + self, + field_name: str, + value: object, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> _RenderedTypeOutput | None: + if not _native_tool_calling_enabled(call, ctx): + return None + if field_name != _tool_call_output_field_name(call.source_signature): + return None + tool_calls = value if isinstance(value, ToolCalls) else ToolCalls.model_validate(value) + return _RenderedTypeOutput(parts=list(tool_calls.to_lm_parts())) + + def _apply_text_parallel_policy(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + if ctx.allow_parallel_tool_calls is not False: + return + field_name = _tool_call_output_field_name(call.render_signature) + if field_name is None: + return + call.render_signature = call.render_signature.with_updated_fields( + field_name, + **ToolCalls.json_schema_extra_for_max_items(1), + ) + + +class _ReasoningTypeHandler(_TypeFeatureHandler): + def prepare(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + reasoning_effort = self._native_reasoning_effort(call, ctx) + if reasoning_effort is None: + return + for field_name, field_info in call.source_signature.output_fields.items(): + if field_info.annotation is Reasoning: + call.merge_config(LMConfig(reasoning=LMReasoningConfig(effort=reasoning_effort))) + call.delete_field(field_name) + + def parse( + self, + values: dict[str, object], + output: LMOutput, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> None: + if self._native_reasoning_effort(call, ctx) is None or not output.reasoning_content: + return + for field_name, field_info in call.source_signature.output_fields.items(): + if field_info.annotation is Reasoning and field_name not in values: + values[field_name] = Reasoning(content=output.reasoning_content) + + def format_output( + self, + field_name: str, + value: object, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> _RenderedTypeOutput | None: + if self._native_reasoning_effort(call, ctx) is None: + return None + field_info = call.source_signature.output_fields.get(field_name) + if field_info is None or field_info.annotation is not Reasoning: + return None + reasoning = value if isinstance(value, Reasoning) else Reasoning.model_validate(value) + return _RenderedTypeOutput(parts=[LMThinkingPart(text=reasoning.content)]) + + @staticmethod + def _native_reasoning_effort(call: _AdapterCallPlan, ctx: _CallContext) -> str | None: + if Reasoning not in ctx.native_response_types: + return None + if not ctx.lm.supports_reasoning: + return None + if "gpt-5" in ctx.lm.model and ctx.lm.model_type == "chat": + return None + + if "reasoning_effort" in call.lm_kwargs: + reasoning_effort = call.lm_kwargs["reasoning_effort"] + elif "reasoning_effort" in ctx.lm_default_kwargs: + reasoning_effort = ctx.lm_default_kwargs["reasoning_effort"] + else: + reasoning_effort = "low" + return reasoning_effort + + +class _CitationsTypeHandler(_TypeFeatureHandler): + def prepare(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + if not self._native_citations_enabled(ctx): + return + for field_name, field_info in call.source_signature.output_fields.items(): + if field_info.annotation is Citations: + call.delete_field(field_name) + + def parse( + self, + values: dict[str, object], + output: LMOutput, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> None: + if not self._native_citations_enabled(ctx) or not output.citations: + return + for field_name, field_info in call.source_signature.output_fields.items(): + if field_info.annotation is Citations and field_name not in values: + values[field_name] = Citations.from_dict_list( + [_lm_part_to_citation_dict(citation) for citation in output.citations] + ) + + def format_output( + self, + field_name: str, + value: object, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> _RenderedTypeOutput | None: + if not self._native_citations_enabled(ctx): + return None + field_info = call.source_signature.output_fields.get(field_name) + if field_info is None or field_info.annotation is not Citations: + return None + citations = value if isinstance(value, Citations) else Citations.model_validate(value) + return _RenderedTypeOutput(parts=[_citation_to_lm_part(citation) for citation in citations.citations]) + + @staticmethod + def _native_citations_enabled(ctx: _CallContext) -> bool: + return Citations in ctx.native_response_types and ctx.lm.model.startswith("anthropic/") + + +def _native_tool_calling_enabled(call: _AdapterCallPlan, ctx: _CallContext) -> bool: + return bool( + ctx.use_native_function_calling + and ctx.lm.supports_function_calling + and _tool_call_output_field_name(call.source_signature) is not None + ) + + +def _tool_input_field_name(signature: type[Any]) -> str | None: + for name, field in signature.input_fields.items(): + if _annotation_includes(field.annotation, Tool): + return name + return None + + +def _tool_call_output_field_name(signature: type[Any]) -> str | None: + for name, field in signature.output_fields.items(): + if _annotation_includes(field.annotation, ToolCalls): + return name + return None + + +def _annotation_includes(annotation: Any, target: type) -> bool: + if annotation is target: + return True + return any(_annotation_includes(arg, target) for arg in get_args(annotation)) + + +def _citation_to_lm_part(citation: Citations.Citation) -> LMCitationPart: + return LMCitationPart( + text=citation.cited_text, + title=citation.document_title, + metadata={ + "type": citation.type, + "document_index": citation.document_index, + "start_char_index": citation.start_char_index, + "end_char_index": citation.end_char_index, + **({"supported_text": citation.supported_text} if citation.supported_text is not None else {}), + }, + ) + + +def _lm_part_to_citation_dict(citation: LMCitationPart) -> dict[str, Any]: + data = dict(citation.metadata) + if citation.text is not None: + data["cited_text"] = citation.text + if citation.title is not None: + data["document_title"] = citation.title + data.setdefault("document_index", 0) + data.setdefault("start_char_index", 0) + data.setdefault("end_char_index", len(data.get("cited_text", ""))) + return data diff --git a/dspy/adapters/_type_runtime.py b/dspy/adapters/_type_runtime.py new file mode 100644 index 0000000000..9c5e5d1985 --- /dev/null +++ b/dspy/adapters/_type_runtime.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Literal + +from pydantic.fields import FieldInfo + +from dspy.core.types import LMConfig, LMMessage, LMOutput, LMPart, LMToolSpec + +if TYPE_CHECKING: + from dspy.adapters.base import Adapter + from dspy.clients.base_lm import BaseLM + from dspy.signatures.signature import Signature + + +@dataclass(frozen=True) +class _CallContext: + adapter: Adapter + use_native_function_calling: bool + allow_parallel_tool_calls: bool | None + native_response_types: tuple[type[object], ...] + lm: BaseLM + lm_kwargs: dict[str, object] = field(default_factory=dict) + lm_default_kwargs: dict[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True) +class _TypeRenderContext: + field_name: str + field_info: FieldInfo + signature: type[Signature] + values: dict[str, object] + adapter: Adapter + call_context: _CallContext + role: Literal["input", "output"] + + +@dataclass(frozen=True) +class _TypeParseContext: + field_name: str + field_info: FieldInfo + signature: type[Signature] + adapter: Adapter + call_context: _CallContext + parts: list[LMPart] + lm_output: LMOutput + + +_OutputParser = Callable[[_TypeParseContext], object] + + +@dataclass +class _AdapterCallPlan: + source_signature: type[Signature] + render_signature: type[Signature] + inputs: dict[str, object] + lm_kwargs: dict[str, object] + messages: list[LMMessage] = field(default_factory=list) + tools: list[LMToolSpec] = field(default_factory=list) + config: LMConfig | None = None + output_parsers: dict[str, _OutputParser] = field(default_factory=dict) + + @classmethod + def from_signature( + cls, + signature: type[Signature], + inputs: dict[str, object], + lm_kwargs: dict[str, object] | None = None, + ) -> _AdapterCallPlan: + return cls( + source_signature=signature, + render_signature=signature, + inputs=dict(inputs), + lm_kwargs=dict(lm_kwargs or {}), + ) + + def delete_field(self, field_name: str) -> None: + if field_name in self.render_signature.fields: + self.render_signature = self.render_signature.delete(field_name) + + def merge_config(self, config: LMConfig | None) -> None: + if config is None: + return + self.config = _merge_lm_config(self.config, config) + + +class _TypeFeatureHandler: + def prepare(self, call: _AdapterCallPlan, ctx: _CallContext) -> None: + pass + + def parse( + self, + values: dict[str, object], + output: LMOutput, + call: _AdapterCallPlan, + ctx: _CallContext, + ) -> None: + pass + + +def _merge_lm_config(left: LMConfig | None, right: LMConfig | None) -> LMConfig | None: + if left is None: + return right + if right is None: + return left + + data = left.model_dump() + right_data = right.model_dump(exclude_none=True, exclude_unset=True) + for key in ("reasoning", "tool_choice", "cache", "prompt_cache"): + if key in right_data and isinstance(data.get(key), dict) and isinstance(right_data[key], dict): + right_data[key] = {**data[key], **right_data[key]} + data.update(right_data) + data["extensions"] = {**left.extensions, **right.extensions} + return LMConfig(**data) diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py index 7520856182..86d3eeae01 100644 --- a/dspy/adapters/base.py +++ b/dspy/adapters/base.py @@ -1,13 +1,42 @@ +import inspect +import json import logging -from typing import Any, get_origin - -import json_repair - +from dataclasses import dataclass +from typing import Any, get_args + +from pydantic.fields import FieldInfo + +from dspy.adapters._legacy_type_markers import ( + _expand_legacy_custom_type_markers_in_lm_message, + _split_legacy_custom_type_text_to_parts, +) +from dspy.adapters._type_feature_handlers import ( + _CitationsTypeHandler, + _ReasoningTypeHandler, + _RenderedTypeOutput, + _ToolTypeHandler, +) +from dspy.adapters._type_runtime import ( + _AdapterCallPlan, + _CallContext, + _merge_lm_config, + _OutputParser, + _TypeFeatureHandler, + _TypeParseContext, + _TypeRenderContext, +) from dspy.adapters.types import History, Type -from dspy.adapters.types.base_type import split_message_content_for_custom_types from dspy.adapters.types.reasoning import Reasoning -from dspy.adapters.types.tool import Tool, ToolCalls -from dspy.clients.base_lm import BaseLM +from dspy.adapters.types.tool import ToolCallResults, ToolCalls +from dspy.adapters.utils import format_field_value, parse_value +from dspy.clients.base_lm import BaseLM, LMCapabilities +from dspy.clients.openai_format import ( + lm_response_from_legacy_outputs, + message_to_openai_chat, + parts_to_openai_content, + to_openai_chat_request, +) +from dspy.core.types import LMConfig, LMMessage, LMOutput, LMPart, LMRequest, LMResponse, LMTextPart from dspy.experimental import Citations from dspy.signatures.signature import Signature from dspy.utils.callback import BaseCallback, with_callbacks @@ -18,6 +47,28 @@ _DEFAULT_NATIVE_RESPONSE_TYPES = [Citations, Reasoning] +@dataclass +class _RenderedAdapterRequest: + request: LMRequest + call_plan: _AdapterCallPlan + context: _CallContext + + @property + def signature(self) -> type[Signature]: + return self.call_plan.render_signature + + +class _AdapterContextLM(BaseLM): + """Minimal BaseLM used when rendering adapter messages without a real LM.""" + + def __init__(self, *, use_native_tool_calls: bool = False): + super().__init__(model="", model_type="chat", temperature=None, max_tokens=None) + self._capabilities = LMCapabilities(function_calling=use_native_tool_calls) + + def get_capabilities(self) -> LMCapabilities: + return self._capabilities + + class Adapter: """Base Adapter class. @@ -40,6 +91,7 @@ def __init__( callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = False, native_response_types: list[type[Type]] | None = None, + allow_parallel_tool_calls: bool | None = None, ): """ Args: @@ -50,11 +102,16 @@ def __init__( or `list[dspy.Tool]` types. Defaults to False. native_response_types: List of output field types that should be handled by native LM features rather than adapter parsing. For example, `dspy.Citations` can be populated directly by citation APIs - (e.g., Anthropic's citation feature). Defaults to `[Citations]`. + (e.g., Anthropic's citation feature). Defaults to `[Citations, Reasoning]`. + allow_parallel_tool_calls: Optional provider/tool-call policy. `False` enforces one call per turn for + native and non-native tool calling; `None` preserves provider/model defaults. """ self.callbacks = callbacks or [] self.use_native_function_calling = use_native_function_calling - self.native_response_types = native_response_types or _DEFAULT_NATIVE_RESPONSE_TYPES + self.native_response_types = list( + _DEFAULT_NATIVE_RESPONSE_TYPES if native_response_types is None else native_response_types + ) + self.allow_parallel_tool_calls = allow_parallel_tool_calls def __init_subclass__(cls, **kwargs) -> None: super().__init_subclass__(**kwargs) @@ -63,118 +120,447 @@ def __init_subclass__(cls, **kwargs) -> None: cls.format = with_callbacks(cls.format) cls.parse = with_callbacks(cls.parse) - def _call_preprocess( + def build_call_context(self, lm: BaseLM, lm_kwargs: dict[str, Any] | None = None) -> _CallContext: + return _CallContext( + adapter=self, + use_native_function_calling=self.use_native_function_calling, + allow_parallel_tool_calls=self.allow_parallel_tool_calls, + native_response_types=tuple(self.native_response_types), + lm=lm, + lm_kwargs=dict(lm_kwargs or {}), + lm_default_kwargs=dict(getattr(lm, "kwargs", {}) or {}), + ) + + def _default_call_context(self, *, use_native_tool_calls: bool = False) -> _CallContext: + return _CallContext( + adapter=self, + use_native_function_calling=use_native_tool_calls, + allow_parallel_tool_calls=self.allow_parallel_tool_calls, + native_response_types=tuple(self.native_response_types), + lm=_AdapterContextLM(use_native_tool_calls=use_native_tool_calls), + ) + + def value_to_lm_parts(self, value: object, field_info: FieldInfo) -> list[LMPart]: + rendered = format_field_value(field_info=field_info, value=value) + return self.coerce_field_payload_to_lm_parts(rendered) + + def coerce_field_payload_to_lm_parts(self, rendered: object) -> list[LMPart]: + if isinstance(rendered, str): + if "<>" in rendered: + return _split_legacy_custom_type_text_to_parts(rendered) + return [LMTextPart(text=rendered)] + if isinstance(rendered, list): + return LMMessage.model_validate({"role": "user", "content": rendered}).parts + if isinstance(rendered, dict): + if "type" in rendered: + return LMMessage.model_validate({"role": "user", "content": [rendered]}).parts + return [LMTextPart(text=json.dumps(rendered, ensure_ascii=False))] + return [LMTextPart(text=str(rendered))] + + def lm_parts_to_text(self, parts: list[LMPart]) -> str: + texts: list[str] = [] + for part in parts: + text = getattr(part, "text", None) + if text is not None: + texts.append(str(text)) + else: + texts.append(str(part)) + return "".join(texts) + + def lm_parts_to_value(self, parts: list[LMPart], field_info: FieldInfo) -> object: + text = self.lm_parts_to_text(parts) + return parse_value(text, field_info.annotation) + + def wrap_input_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + raise NotImplementedError + + def wrap_output_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + raise NotImplementedError + + def parse_output_fields_to_parts( + self, + output: LMOutput, + signature: type[Signature], + ) -> dict[str, list[LMPart]]: + parsed = self.parse(signature, output.text or "") + return { + field_name: self.value_to_lm_parts(parsed[field_name], field_info) + for field_name, field_info in signature.output_fields.items() + if field_name in parsed + } + + def _parts_to_message_content(self, parts: list[LMPart]) -> str | list[dict[str, Any]]: + return parts_to_openai_content(parts) + + def _make_type_render_context( + self, + *, + field_name: str, + field_info: FieldInfo, + signature: type[Signature], + values: dict[str, Any], + role: str, + context: _CallContext | None, + ) -> _TypeRenderContext: + return _TypeRenderContext( + field_name=field_name, + field_info=field_info, + signature=signature, + values=values, + adapter=self, + call_context=context or self._current_call_context(), + role=role, + ) + + def _current_call_context(self) -> _CallContext: + return getattr(self, "_call_context", None) or self._default_call_context() + + def _default_render_type_value( + self, + *, + field_name: str, + field_info: FieldInfo, + signature: type[Signature], + values: dict[str, Any], + value: Any, + role: str, + context: _CallContext | None = None, + ) -> list[LMPart] | None: + annotation = self._type_annotation_class(field_info.annotation) + if annotation is not None and not isinstance(value, annotation): + try: + value = annotation.model_validate(value) + except Exception: + pass + + if not isinstance(value, Type): + return None + + render_ctx = self._make_type_render_context( + field_name=field_name, + field_info=field_info, + signature=signature, + values=values, + role=role, + context=context, + ) + if role == "input": + return value.default_render_input(render_ctx) + return value.default_render_output(render_ctx) + + def _type_annotation_class(self, annotation: Any) -> type[Type] | None: + try: + if isinstance(annotation, type) and issubclass(annotation, Type): + return annotation + except TypeError: + return None + + for arg in get_args(annotation): + annotation_class = self._type_annotation_class(arg) + if annotation_class is not None: + return annotation_class + return None + + def _merge_config_kwargs(self, kwargs: dict[str, Any], config: LMConfig | None) -> dict[str, Any]: + if config is None: + return kwargs + base_config = LMConfig.from_kwargs(**kwargs) + merged_config = _merge_lm_config(base_config, config) + return merged_config.model_dump(exclude_none=True) + + def _merge_lm_config(self, left: LMConfig | None, right: LMConfig | None) -> LMConfig | None: + return _merge_lm_config(left, right) + + def _render_request( self, lm: BaseLM, lm_kwargs: dict[str, Any], signature: type[Signature], + demos: list[dict[str, Any]], inputs: dict[str, Any], - ) -> type[Signature]: - if self.use_native_function_calling: - tool_call_input_field_name = self._get_tool_call_input_field_name(signature) - tool_call_output_field_name = self._get_tool_call_output_field_name(signature) - - if tool_call_output_field_name and tool_call_input_field_name is None: - raise ValueError( - f"You provided an output field {tool_call_output_field_name} to receive the tool calls information, " - "but did not provide any tools as the input. Please provide a list of tools as the input by adding an " - "input field with type `list[dspy.Tool]`." - ) - - if tool_call_output_field_name and lm.supports_function_calling: - tools = inputs[tool_call_input_field_name] - tools = tools if isinstance(tools, list) else [tools] + ) -> _RenderedAdapterRequest: + """Render the full signature into a normalized LM request.""" + context = self.build_call_context(lm, lm_kwargs) + call_plan = _AdapterCallPlan.from_signature(signature, inputs, lm_kwargs) + self._prepare_call_plan(call_plan, context) + + request_kwargs = self._prepare_request_kwargs(lm, call_plan.lm_kwargs, signature, call_plan.render_signature, context) + tools = list(call_plan.tools) + if "tools" in request_kwargs: + tools.extend(request_kwargs.pop("tools") or []) + request_kwargs = self._merge_config_kwargs(request_kwargs, call_plan.config) + if not tools: + request_kwargs.pop("tool_choice", None) + + messages = self._format_request_messages(call_plan, demos, context) + request = LMRequest.from_call( + model=lm.model, + messages=self._coerce_lm_messages(messages), + tools=tools, + **request_kwargs, + ) + return _RenderedAdapterRequest(request=request, call_plan=call_plan, context=context) - lm_tools = [tool.format_as_litellm_function_call() for tool in tools] + def _type_feature_handlers(self) -> list[_TypeFeatureHandler]: + return [_ToolTypeHandler(), _ReasoningTypeHandler(), _CitationsTypeHandler()] - lm_kwargs["tools"] = lm_tools + def _prepare_call_plan(self, call: _AdapterCallPlan, context: _CallContext) -> None: + for field_name, field_info in call.source_signature.output_fields.items(): + annotation = self._type_annotation_class(field_info.annotation) + if annotation is not None: + call.output_parsers[field_name] = annotation.default_parse_output - signature_for_native_function_calling = signature.delete(tool_call_output_field_name) - signature_for_native_function_calling = signature_for_native_function_calling.delete( - tool_call_input_field_name - ) + for handler in self._type_feature_handlers(): + handler.prepare(call, context) - return signature_for_native_function_calling + @staticmethod + def _default_signature_instructions(signature: type[Signature]) -> str: + inputs = ", ".join([f"`{field}`" for field in signature.input_fields]) + outputs = ", ".join([f"`{field}`" for field in signature.output_fields]) + if not outputs: + return f"Given the fields {inputs}, follow the instructions." + return f"Given the fields {inputs}, produce the fields {outputs}." - # Handle custom types that use native LM features, e.g., reasoning, citations, etc. - for name, field in signature.output_fields.items(): - if ( - isinstance(field.annotation, type) - and field.annotation in self.native_response_types - and issubclass(field.annotation, Type) - ): - signature = field.annotation.adapt_to_native_lm_feature(signature, name, lm, lm_kwargs) + def _format_request_messages( + self, + call_plan: _AdapterCallPlan, + demos: list[dict[str, Any]], + context: _CallContext, + ) -> list[LMMessage | dict[str, Any]]: + format_signature = inspect.signature(self.format) + accepts_kwargs = any(param.kind == inspect.Parameter.VAR_KEYWORD for param in format_signature.parameters.values()) + kwargs: dict[str, Any] = {} + if accepts_kwargs or "use_native_tool_calls" in format_signature.parameters: + kwargs["use_native_tool_calls"] = bool(call_plan.tools) + if accepts_kwargs or "context" in format_signature.parameters: + kwargs["context"] = context + if accepts_kwargs or "source_signature" in format_signature.parameters: + kwargs["source_signature"] = call_plan.source_signature + if accepts_kwargs or "prepared_messages" in format_signature.parameters: + kwargs["prepared_messages"] = call_plan.messages + sentinel = object() + previous_context = getattr(self, "_call_context", sentinel) + self._call_context = context + try: + messages = self.format(call_plan.render_signature, demos, call_plan.inputs, **kwargs) + finally: + if previous_context is sentinel: + del self._call_context + else: + self._call_context = previous_context - return signature + if not (accepts_kwargs or "prepared_messages" in format_signature.parameters): + messages = [*messages, *call_plan.messages] + return messages - def _call_postprocess( + def _prepare_request_kwargs( self, - processed_signature: type[Signature], - original_signature: type[Signature], - outputs: list[dict[str, Any] | str], lm: BaseLM, lm_kwargs: dict[str, Any], + source_signature: type[Signature], + render_signature: type[Signature], + context: _CallContext, + ) -> dict[str, Any]: + return dict(lm_kwargs) + + def _call_lm(self, lm: BaseLM, request: LMRequest) -> LMResponse: + """Call current `BaseLM` through the normalized request/response boundary. + + TODO(language-models): When `BaseLM` is replaced by/updated to the + normalized `BaseLM.forward(request: LMRequest) -> LMResponse` contract, + remove this compatibility shim and let adapters call the normalized LM + entry point directly. The OpenAI-shaped compatibility kwargs should live + only inside concrete LM backends. + """ + data = self._legacy_call_kwargs(request) + outputs = lm(messages=data.pop("messages"), **data) + return self._normalize_legacy_outputs(outputs, request) + + async def _acall_lm(self, lm: BaseLM, request: LMRequest) -> LMResponse: + """Async variant of `_call_lm`. + + TODO(language-models): Same transitional boundary as `_call_lm()`; this + should eventually call a normalized async LM method directly. + """ + data = self._legacy_call_kwargs(request) + outputs = await lm.acall(messages=data.pop("messages"), **data) + return self._normalize_legacy_outputs(outputs, request) + + def _legacy_call_kwargs(self, request: LMRequest) -> dict[str, Any]: + # TODO(language-models): Current `BaseLM` expects OpenAI/LiteLLM-shaped + # chat kwargs. We intentionally use `dspy.clients.openai_format` here so + # the conversion code lives in the future LM/client layer, not in + # adapters. Remove this adapter helper once `BaseLM` accepts `LMRequest`. + data = to_openai_chat_request(request) + data.pop("model", None) + # TODO(language-models): `cache` and `rollout_id` are DSPy BaseLM + # execution controls, not provider request fields. The future + # normalized LM base should own them before provider-format conversion. + if request.config.cache is not None: + if request.config.cache.enabled is not None: + data["cache"] = request.config.cache.enabled + if request.config.cache.rollout_id is not None: + data["rollout_id"] = request.config.cache.rollout_id + return data + + def _coerce_lm_messages(self, messages: list[LMMessage | dict[str, Any]]) -> list[LMMessage]: + """Normalize subclass `format()` output before the LM boundary. + + TODO(adapters-normalized-rendering): Adapter `format()` methods still + return OpenAI-chat-shaped dictionaries. This coercion is the bridge until + adapters render `LMMessage` / `LMPart` directly. + """ + return [ + _expand_legacy_custom_type_markers_in_lm_message( + message if isinstance(message, LMMessage) else self._chat_dict_to_lm_message(message) + ) + for message in messages + ] + + def _chat_dict_to_lm_message(self, message: dict[str, Any]) -> LMMessage: + try: + return LMMessage(**message) + except Exception: + # TODO(legacy-custom-types): Unknown OpenAI content blocks are + # temporarily preserved as `legacy_content_block` metadata on an + # empty text part so `openai_format` can round-trip them back to + # current BaseLM calls. Replace this with either explicit opaque + # provider parts or remove it when marker-based custom type + # serialization is retired. + message = dict(message) + content = message.get("content") + if isinstance(content, list): + sanitized = [] + supported = {"text", "image_url", "input_audio", "file", "document", "video"} + for block in content: + if isinstance(block, dict) and block.get("type") in supported: + sanitized.append(block) + elif isinstance(block, dict): + sanitized.append({"type": "text", "text": "", "metadata": {"legacy_content_block": block}}) + else: + sanitized.append({"type": "text", "text": json.dumps(block, ensure_ascii=False)}) + message["content"] = sanitized + return LMMessage(**message) + + def _normalize_legacy_outputs(self, outputs: list[dict[str, Any] | str | None], request: LMRequest) -> LMResponse: + """Convert current `BaseLM` outputs into a normalized `LMResponse`. + + TODO(language-models): Current `BaseLM` returns `list[str | dict | None]`. + Future LMs should return `LMResponse` directly, making this method a + compatibility-only path for old/custom LMs. + """ + return lm_response_from_legacy_outputs(outputs, request) + + def _parse_response( + self, + source_signature: type[Signature], + call_plan: _AdapterCallPlan, + response: LMResponse, + context: _CallContext, ) -> list[dict[str, Any]]: + """Parse a normalized LM response into dictionaries for the source signature.""" values = [] - tool_call_output_field_name = self._get_tool_call_output_field_name(original_signature) - - for output in outputs: - output_logprobs = None - tool_calls = None - text = output - - if isinstance(output, dict): - text = output["text"] - output_logprobs = output.get("logprobs") - tool_calls = output.get("tool_calls") - - if text: - value = self.parse(processed_signature, text) - for field_name in original_signature.output_fields.keys(): - if field_name not in value: - # We need to set the field not present in the processed signature to None for consistency. - value[field_name] = None - elif tool_calls and tool_call_output_field_name: - value = {} - for field_name in original_signature.output_fields.keys(): - value[field_name] = None - else: + for output in response.outputs: + if output.metadata.get("empty_legacy_outputs"): + continue + + field_parts: dict[str, list[LMPart]] = {} + has_text_output = bool(output.text and call_plan.render_signature.output_fields) + if has_text_output: + field_parts = self.parse_output_fields_to_parts(output, call_plan.render_signature) + + value: dict[str, Any] = {} + for field_name, field_info in call_plan.render_signature.output_fields.items(): + if field_name in field_parts: + value[field_name] = self._parse_field_from_parts( + field_name, + field_info, + call_plan.render_signature, + field_parts[field_name], + output, + context, + call_plan.output_parsers.get(field_name), + ) + + before_type_handler_count = len(value) + for handler in self._type_feature_handlers(): + handler.parse(value, output, call_plan, context) + parsed_by_type_handler = len(value) > before_type_handler_count + + for field_name in source_signature.output_fields: + value.setdefault(field_name, None) + + if not (field_parts or parsed_by_type_handler): raise AdapterParseError( adapter_name=type(self).__name__, - signature=original_signature, - lm_response=str(output), + signature=source_signature, + lm_response=str(output.to_output_dict()), message="The LM returned an empty or null response.", ) - if tool_calls and tool_call_output_field_name: - tool_calls = [ - { - "name": v["function"]["name"], - "args": json_repair.loads(v["function"]["arguments"]), - } - for v in tool_calls - ] - value[tool_call_output_field_name] = ToolCalls.from_dict_list(tool_calls) - - # Parse custom types that does not rely on the `Adapter.parse()` method - for name, field in original_signature.output_fields.items(): - if ( - isinstance(field.annotation, type) - and field.annotation in self.native_response_types - and issubclass(field.annotation, Type) - ): - parsed_value = field.annotation.parse_lm_response(output) - if parsed_value is not None: - value[name] = parsed_value - - if output_logprobs: - value["logprobs"] = output_logprobs + if output.logprobs is not None: + value["logprobs"] = output.logprobs + self._validate_tool_call_parallel_policy(source_signature, value) values.append(value) return values + def _parse_field_from_parts( + self, + field_name: str, + field_info: FieldInfo, + signature: type[Signature], + parts: list[LMPart], + output: LMOutput, + context: _CallContext, + output_parser: _OutputParser | None = None, + ) -> Any: + if output_parser is not None: + return output_parser( + _TypeParseContext( + field_name=field_name, + field_info=field_info, + signature=signature, + adapter=self, + call_context=context, + parts=parts, + lm_output=output, + ) + ) + + annotation = self._type_annotation_class(field_info.annotation) + if annotation is not None: + return annotation.default_parse_output( + _TypeParseContext( + field_name=field_name, + field_info=field_info, + signature=signature, + adapter=self, + call_context=context, + parts=parts, + lm_output=output, + ) + ) + return self.lm_parts_to_value(parts, field_info) + + def _validate_tool_call_parallel_policy(self, signature: type[Signature], value: dict[str, Any]) -> None: + if self.allow_parallel_tool_calls is not False: + return + + tool_call_output_field_name = self._get_tool_call_output_field_name(signature) + if not tool_call_output_field_name or tool_call_output_field_name not in value: + return + + tool_calls = value[tool_call_output_field_name] + if tool_calls is None: + return + if not isinstance(tool_calls, ToolCalls): + tool_calls = ToolCalls.model_validate(tool_calls) + value[tool_call_output_field_name] = tool_calls + tool_calls.validate_max_items(1) + def __call__( self, lm: BaseLM, @@ -199,11 +585,9 @@ def __call__( List of dictionaries representing parsed LM responses. Each dictionary contains keys matching the signature's output field names. For multiple generations (n > 1), returns multiple dictionaries. """ - processed_signature = self._call_preprocess(lm, lm_kwargs, signature, inputs) - inputs = self.format(processed_signature, demos, inputs) - - outputs = lm(messages=inputs, **lm_kwargs) - return self._call_postprocess(processed_signature, signature, outputs, lm, lm_kwargs) + rendered = self._render_request(lm, lm_kwargs, signature, demos, inputs) + response = self._call_lm(lm, rendered.request) + return self._parse_response(signature, rendered.call_plan, response, rendered.context) async def acall( self, @@ -213,17 +597,20 @@ async def acall( demos: list[dict[str, Any]], inputs: dict[str, Any], ) -> list[dict[str, Any]]: - processed_signature = self._call_preprocess(lm, lm_kwargs, signature, inputs) - inputs = self.format(processed_signature, demos, inputs) - - outputs = await lm.acall(messages=inputs, **lm_kwargs) - return self._call_postprocess(processed_signature, signature, outputs, lm, lm_kwargs) + rendered = self._render_request(lm, lm_kwargs, signature, demos, inputs) + response = await self._acall_lm(lm, rendered.request) + return self._parse_response(signature, rendered.call_plan, response, rendered.context) def format( self, signature: type[Signature], demos: list[dict[str, Any]], inputs: dict[str, Any], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + source_signature: type[Signature] | None = None, + prepared_messages: list[LMMessage] | None = None, ) -> list[dict[str, Any]]: """Format the input messages for the LM call. @@ -264,37 +651,413 @@ def format( Returns: A list of multiturn messages as expected by the LM. """ + return [ + message_to_openai_chat(message) + for message in self.render_messages( + signature, + demos, + inputs, + use_native_tool_calls=use_native_tool_calls, + context=context, + source_signature=source_signature, + prepared_messages=prepared_messages, + ) + ] + + def render_messages( + self, + signature: type[Signature], + demos: list[dict[str, Any]], + inputs: dict[str, Any], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + source_signature: type[Signature] | None = None, + prepared_messages: list[LMMessage] | None = None, + ) -> list[LMMessage]: + """Render adapter messages as normalized LM messages.""" + context = context or self._default_call_context(use_native_tool_calls=use_native_tool_calls) + source_signature = source_signature or signature inputs_copy = dict(inputs) - # If the signature and inputs have conversation history, we need to format the conversation history and - # remove the history field from the signature. - history_field_name = self._get_history_field_name(signature) + render_signature = signature + conversation_history: list[LMMessage] = [] + history_field_name = self._get_history_field_name(source_signature) if history_field_name: - # In order to format the conversation history, we need to remove the history field from the signature. - signature_without_history = signature.delete(history_field_name) - conversation_history = self.format_conversation_history( - signature_without_history, - history_field_name, + if history_field_name in render_signature.fields: + render_signature = render_signature.delete(history_field_name) + if source_signature.instructions == self._default_signature_instructions(source_signature): + instruction_signature = source_signature.delete(history_field_name) + render_signature = render_signature.with_instructions( + self._default_signature_instructions(instruction_signature) + ) + history_obj = inputs_copy.pop(history_field_name, None) + if history_obj is not None: + conversation_history = self.format_history( + history_obj, + source_signature.delete(history_field_name), + context=context, + ) + + messages: list[LMMessage | dict[str, Any]] = [ + {"role": "system", "content": self.format_system_message(render_signature)} + ] + messages.extend( + self.format_demos( + source_signature.delete(history_field_name) if history_field_name else source_signature, + demos, + context=context, + ) + ) + messages.extend(conversation_history) + messages.extend(prepared_messages or []) + messages.extend( + self._format_input_messages( + render_signature, inputs_copy, + main_request=True, + context=context, ) + ) + + return self._coerce_lm_messages(messages) + + def format_history( + self, + history: History, + signature: type[Signature], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + ) -> list[LMMessage]: + """Render stored field history using this adapter's field format.""" + context = context or self._default_call_context(use_native_tool_calls=use_native_tool_calls) + messages: list[LMMessage] = [] + for message_idx, entry in enumerate(history.messages): + if not isinstance(entry, dict): + continue + + input_values = {key: value for key, value in entry.items() if key in signature.input_fields} + tool_result_values = { + key: value + for key, value in input_values.items() + if self._coerce_tool_call_results_value(value, signature.input_fields.get(key)) is not None + } + output_values = {key: value for key, value in entry.items() if key in signature.output_fields} + unknown_values = { + key: value for key, value in entry.items() if key not in input_values and key not in output_values + } + + turn_call = _AdapterCallPlan.from_signature(signature, input_values, context.lm_kwargs) + self._prepare_call_plan(turn_call, context) + + regular_input_values = { + key: turn_call.inputs[key] + for key in turn_call.render_signature.input_fields + if key in turn_call.inputs and key not in tool_result_values + } + if regular_input_values: + messages.extend( + self._format_input_messages( + turn_call.render_signature, + regular_input_values, + main_request=False, + context=context, + ) + ) + + assistant_values = dict(output_values) + if not turn_call.messages: + assistant_values.update(unknown_values) + + if assistant_values: + messages.extend( + self._format_output_messages_for_call( + turn_call, + assistant_values, + message_idx, + context=context, + ) + ) + + messages.extend(turn_call.messages) + + fallback_tool_result_values = { + key: value + for key, value in tool_result_values.items() + if key in turn_call.render_signature.input_fields and not turn_call.messages + } + if fallback_tool_result_values: + messages.extend( + self._format_input_messages( + turn_call.render_signature, + fallback_tool_result_values, + main_request=False, + context=context, + ) + ) + + if (turn_call.messages or fallback_tool_result_values) and unknown_values: + messages.extend( + self._format_output_messages_for_call( + turn_call, + unknown_values, + message_idx, + context=context, + ) + ) - messages = [] - system_message = self.format_system_message(signature) - messages.append({"role": "system", "content": system_message}) - messages.extend(self.format_demos(signature, demos)) - if history_field_name: - # Conversation history and current input - content = self.format_user_message_content(signature_without_history, inputs_copy, main_request=True) - messages.extend(conversation_history) - messages.append({"role": "user", "content": content}) - else: - # Only current input - content = self.format_user_message_content(signature, inputs_copy, main_request=True) - messages.append({"role": "user", "content": content}) - - messages = split_message_content_for_custom_types(messages) return messages + def _format_input_messages( + self, + signature: type[Signature], + inputs: dict[str, Any], + *, + main_request: bool, + context: _CallContext | None = None, + prefix: str = "", + suffix: str = "", + ) -> list[LMMessage]: + context = context or self._current_call_context() + regular_inputs = self._drop_absent_optional_inputs(signature, dict(inputs)) + if self._should_render_input_as_parts(signature, regular_inputs, context): + parts: list[LMPart] = [] + if prefix: + parts.append(LMTextPart(text=prefix)) + field_seen = False + for key, field_info in signature.input_fields.items(): + if key not in regular_inputs: + continue + if field_seen: + parts.append(LMTextPart(text="\n\n")) + value_parts = self._default_render_type_value( + field_name=key, + field_info=field_info, + signature=signature, + values=regular_inputs, + value=regular_inputs[key], + role="input", + context=context, + ) + if value_parts is None: + value_parts = self.value_to_lm_parts(regular_inputs[key], field_info) + parts.extend(self.wrap_input_field_parts(key, value_parts)) + field_seen = True + if main_request: + output_requirements_fn = getattr(self, "user_message_output_requirements", lambda _signature: None) + output_requirements = output_requirements_fn(signature) + if output_requirements is not None: + if parts: + parts.append(LMTextPart(text="\n\n")) + parts.append(LMTextPart(text=output_requirements)) + if suffix: + parts.append(LMTextPart(text=suffix)) + parts = self._merge_adjacent_text_parts(parts) + return [LMMessage(role="user", parts=parts)] if parts else [] + + content = self.format_user_message_content( + signature, + regular_inputs, + prefix=prefix, + suffix=suffix, + main_request=main_request, + ) + if self._has_content(content): + return [self._content_message("user", content)] + return [] + + def _format_output_messages_for_call( + self, + call: _AdapterCallPlan, + outputs: dict[str, Any], + message_idx: int, + *, + context: _CallContext | None = None, + missing_field_message: str | None = "Not supplied for this conversation history message. ", + ) -> list[LMMessage]: + context = context or self._current_call_context() + regular_outputs = {} + native_parts: list[LMPart] = [] + native_messages: list[LMMessage] = [] + + for key, value in outputs.items(): + if key in call.render_signature.output_fields: + regular_outputs[key] = value + continue + + rendered_type = self._format_type_feature_output(key, value, call, context) + if rendered_type is not None and rendered_type.consumed: + native_parts.extend(rendered_type.parts) + native_messages.extend(rendered_type.messages) + for message in rendered_type.messages: + if message.role == "assistant": + native_parts.extend(message.parts) + continue + regular_outputs[key] = value + + if native_parts: + content = self._format_native_output_content( + call.render_signature, + regular_outputs, + missing_field_message=missing_field_message, + ) + parts: list[LMPart] = [] + if content: + parts.append(LMTextPart(text=content)) + parts.extend(native_parts) + return [LMMessage(role="assistant", parts=parts)] + + if native_messages: + return native_messages + + return [ + self._content_message( + "assistant", + self._format_output_content( + call.render_signature, + regular_outputs, + missing_field_message=missing_field_message, + ), + ) + ] + + def _format_type_feature_output( + self, + field_name: str, + value: object, + call: _AdapterCallPlan, + context: _CallContext, + ) -> _RenderedTypeOutput | None: + for handler in self._type_feature_handlers(): + formatter = getattr(handler, "format_output", None) + if formatter is None: + continue + rendered_type = formatter(field_name, value, call, context) + if rendered_type is not None: + return rendered_type + return None + + def _format_output_content( + self, + signature: type[Signature], + outputs: dict[str, Any], + *, + missing_field_message: str | None, + ) -> str: + signature_outputs = {key: value for key, value in outputs.items() if key in signature.output_fields} + unknown_outputs = {key: value for key, value in outputs.items() if key not in signature.output_fields} + if signature_outputs and not unknown_outputs: + return self.format_assistant_message_content( + signature, + signature_outputs, + missing_field_message=missing_field_message, + ) + + sections = [] + if signature_outputs: + sections.append( + self.format_assistant_message_content( + signature, + signature_outputs, + missing_field_message=missing_field_message, + ).strip() + ) + + for key, value in unknown_outputs.items(): + sections.append(f"[[ ## {key} ## ]]\n{self._format_value(value)}") + + sections.append("[[ ## completed ## ]]") + return "\n\n".join(section for section in sections if section) + + def _format_native_output_content( + self, + signature: type[Signature], + outputs: dict[str, Any], + *, + missing_field_message: str | None, + ) -> str | None: + if not outputs: + return None + if len(outputs) == 1: + return str(next(iter(outputs.values()))) + return self._format_output_content(signature, outputs, missing_field_message=missing_field_message) + + @staticmethod + def _drop_absent_optional_inputs(signature: type[Signature], inputs: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in inputs.items() + if not (value is None and key in signature.input_fields and signature.input_fields[key].default is None) + } + + def _should_render_input_as_parts( + self, + signature: type[Signature], + inputs: dict[str, Any], + context: _CallContext, + ) -> bool: + for key, value in inputs.items(): + field_info = signature.input_fields.get(key) + if field_info is None: + continue + parts = self._default_render_type_value( + field_name=key, + field_info=field_info, + signature=signature, + values=inputs, + value=value, + role="input", + context=context, + ) + if parts is not None and any(not isinstance(part, LMTextPart) for part in parts): + return True + return False + + @staticmethod + def _merge_adjacent_text_parts(parts: list[LMPart]) -> list[LMPart]: + merged: list[LMPart] = [] + for part in parts: + if ( + isinstance(part, LMTextPart) + and merged + and isinstance(merged[-1], LMTextPart) + and not part.metadata + and not merged[-1].metadata + ): + merged[-1] = LMTextPart(text=merged[-1].text + part.text) + else: + merged.append(part) + return merged + + @staticmethod + def _coerce_tool_call_results_value(field_value: Any, field_info: Any = None) -> ToolCallResults | None: + if field_value is None: + return None + if isinstance(field_value, ToolCallResults): + return field_value + + annotation = getattr(field_info, "annotation", None) + if annotation is ToolCallResults or ToolCallResults in get_args(annotation): + return ToolCallResults.model_validate(field_value) + return None + + @staticmethod + def _format_value(value: Any) -> str: + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return str(value) + + @staticmethod + def _has_content(content: Any) -> bool: + if isinstance(content, str): + return bool(content.strip()) + return bool(content) + + @staticmethod + def _content_message(role: str, content: Any) -> LMMessage: + return LMMessage.model_validate({"role": role, "content": content}) + def format_system_message(self, signature: type[Signature]) -> str: """Format the system message for the LM call. @@ -393,7 +1156,14 @@ def format_assistant_message_content( """ raise NotImplementedError - def format_demos(self, signature: type[Signature], demos: list[dict[str, Any]]) -> list[dict[str, Any]]: + def format_demos( + self, + signature: type[Signature], + demos: list[dict[str, Any]], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + ) -> list[LMMessage | dict[str, Any]]: """Format the few-shot examples. This method formats the few-shot examples as multiturn messages. @@ -406,77 +1176,83 @@ def format_demos(self, signature: type[Signature], demos: list[dict[str, Any]]) Returns: A list of multiturn messages. """ - complete_demos = [] - incomplete_demos = [] + context = context or self._default_call_context(use_native_tool_calls=use_native_tool_calls) + active_fields = signature.fields + messages = [] + incomplete_demo_prefix = "This is an example of the task, though some input or output fields are not supplied." for demo in demos: - # Check if all fields are present and not None - is_complete = all(k in demo and demo[k] is not None for k in signature.fields) - - # Check if demo has at least one input and one output field + is_complete = all(k in demo and demo[k] is not None for k in active_fields) has_input = any(k in demo for k in signature.input_fields) has_output = any(k in demo for k in signature.output_fields) + if not (is_complete or (has_input and has_output)): + continue - if is_complete: - complete_demos.append(demo) - elif has_input and has_output: - # We only keep incomplete demos that have at least one input and one output field - incomplete_demos.append(demo) - - messages = [] - - incomplete_demo_prefix = "This is an example of the task, though some input or output fields are not supplied." - for demo in incomplete_demos: - messages.append( - { - "role": "user", - "content": self.format_user_message_content(signature, demo, prefix=incomplete_demo_prefix), - } + demo_call = _AdapterCallPlan.from_signature( + signature, + {key: demo[key] for key in signature.input_fields if key in demo}, + context.lm_kwargs, ) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content( - signature, demo, missing_field_message="Not supplied for this particular example. " - ), - } + self._prepare_call_plan(demo_call, context) + demo_inputs = { + key: demo_call.inputs[key] + for key in demo_call.render_signature.input_fields + if key in demo_call.inputs + } + demo_outputs = {key: demo[key] for key in signature.output_fields if key in demo} + messages.extend( + self._format_input_messages( + demo_call.render_signature, + demo_inputs, + main_request=False, + context=context, + prefix="" if is_complete else incomplete_demo_prefix, + ) ) - - for demo in complete_demos: - messages.append({"role": "user", "content": self.format_user_message_content(signature, demo)}) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content( - signature, demo, missing_field_message="Not supplied for this conversation history message. " + messages.extend( + self._format_output_messages_for_call( + demo_call, + demo_outputs, + len(messages), + context=context, + missing_field_message=( + "Not supplied for this conversation history message. " + if is_complete + else "Not supplied for this particular example. " ), - } + ) ) + messages.extend(demo_call.messages) return messages - def _get_history_field_name(self, signature: type[Signature]) -> bool: + def _get_history_field_name(self, signature: type[Signature]) -> str | None: for name, field in signature.input_fields.items(): if field.annotation == History: return name return None - def _get_tool_call_input_field_name(self, signature: type[Signature]) -> bool: - for name, field in signature.input_fields.items(): - # Look for annotation `list[dspy.Tool]` or `dspy.Tool` - origin = get_origin(field.annotation) - if origin is list and field.annotation.__args__[0] == Tool: - return name - if field.annotation == Tool: - return name - return None + def _get_tool_call_output_field_name(self, signature: type[Signature]) -> str | None: + return self._find_tool_call_output_field_name(signature) - def _get_tool_call_output_field_name(self, signature: type[Signature]) -> bool: + @classmethod + def _find_tool_call_output_field_name(cls, signature: type[Signature]) -> str | None: for name, field in signature.output_fields.items(): - if field.annotation == ToolCalls: + if cls._annotation_includes_static(field.annotation, ToolCalls): return name return None + @classmethod + def _annotation_includes_static(cls, annotation: Any, target: type) -> bool: + if annotation is target: + return True + return any(cls._annotation_includes_static(arg, target) for arg in get_args(annotation)) + + def force_tool_call_config(self, tool_name: str) -> dict[str, Any]: + if not self.use_native_function_calling: + return {} + return {"tool_choice": {"mode": "required", "allowed": [tool_name]}} + def format_conversation_history( self, signature: type[Signature], @@ -500,25 +1276,10 @@ def format_conversation_history( if conversation_history is None: return [] - messages = [] - for message in conversation_history: - messages.append( - { - "role": "user", - "content": self.format_user_message_content(signature, message), - } - ) - messages.append( - { - "role": "assistant", - "content": self.format_assistant_message_content(signature, message), - } - ) - # Remove the history field from the inputs del inputs[history_field_name] - - return messages + history = History(messages=conversation_history) + return [message_to_openai_chat(message) for message in self.format_history(history, signature)] def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: """Parse the LM output into a dictionary of the output fields. diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py index 016c885d63..90779a9260 100644 --- a/dspy/adapters/chat_adapter.py +++ b/dspy/adapters/chat_adapter.py @@ -13,6 +13,7 @@ translate_field_type, ) from dspy.clients.base_lm import BaseLM +from dspy.core.types import LMOutput, LMPart, LMTextPart from dspy.signatures.signature import Signature from dspy.utils.callback import BaseCallback from dspy.utils.exceptions import AdapterParseError, ContextWindowExceededError @@ -42,6 +43,7 @@ def __init__( callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = False, native_response_types: list[type[type]] | None = None, + allow_parallel_tool_calls: bool | None = None, use_json_adapter_fallback: bool = True, ): """ @@ -49,6 +51,7 @@ def __init__( callbacks: List of callback functions to execute during adapter methods. use_native_function_calling: Whether to enable native function calling capabilities. native_response_types: List of output field types handled by native LM features. + allow_parallel_tool_calls: Whether to allow multiple tool calls in one LM turn. use_json_adapter_fallback: Whether to automatically fallback to JSONAdapter if the ChatAdapter fails. If True, when an error occurs (except ContextWindowExceededError), the adapter will retry using JSONAdapter. Defaults to True. @@ -57,6 +60,7 @@ def __init__( callbacks=callbacks, use_native_function_calling=use_native_function_calling, native_response_types=native_response_types, + allow_parallel_tool_calls=allow_parallel_tool_calls, ) self.use_json_adapter_fallback = use_json_adapter_fallback @@ -82,7 +86,12 @@ def __call__( # On context window exceeded error, already using JSONAdapter, or use_json_adapter_fallback is False # we don't want to retry with a different adapter. Raise the original error instead of the fallback error. raise e - return JSONAdapter()(lm, lm_kwargs, signature, demos, inputs) + return JSONAdapter( + callbacks=self.callbacks, + use_native_function_calling=self.use_native_function_calling, + native_response_types=self.native_response_types, + allow_parallel_tool_calls=self.allow_parallel_tool_calls, + )(lm, lm_kwargs, signature, demos, inputs) async def acall( self, @@ -106,13 +115,18 @@ async def acall( # On context window exceeded error, already using JSONAdapter, or use_json_adapter_fallback is False # we don't want to retry with a different adapter. Raise the original error instead of the fallback error. raise e - return await JSONAdapter().acall(lm, lm_kwargs, signature, demos, inputs) + return await JSONAdapter( + callbacks=self.callbacks, + use_native_function_calling=self.use_native_function_calling, + native_response_types=self.native_response_types, + allow_parallel_tool_calls=self.allow_parallel_tool_calls, + ).acall(lm, lm_kwargs, signature, demos, inputs) def format_field_description(self, signature: type[Signature]) -> str: - return ( - f"Your input fields are:\n{get_field_description_string(signature.input_fields)}\n" - f"Your output fields are:\n{get_field_description_string(signature.output_fields)}" - ) + description = f"Your input fields are:\n{get_field_description_string(signature.input_fields)}" + if signature.output_fields: + description += f"\nYour output fields are:\n{get_field_description_string(signature.output_fields)}" + return description def format_field_structure(self, signature: type[Signature]) -> str: """ @@ -132,8 +146,9 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo]): ) parts.append(format_signature_fields_for_instructions(signature.input_fields)) - parts.append(format_signature_fields_for_instructions(signature.output_fields)) - parts.append("[[ ## completed ## ]]\n") + if signature.output_fields: + parts.append(format_signature_fields_for_instructions(signature.output_fields)) + parts.append("[[ ## completed ## ]]\n") return "\n\n".join(parts).strip() def format_task_description(self, signature: type[Signature]) -> str: @@ -164,7 +179,7 @@ def format_user_message_content( messages.append(suffix) return "\n\n".join(messages).strip() - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: """Returns a simplified format reminder for the language model. In chat-based interactions, language models may lose track of the required output format @@ -181,6 +196,8 @@ def user_message_output_requirements(self, signature: type[Signature]) -> str: This is a more lightweight version of `format_field_structure` specifically designed for inline reminders within chat messages. """ + if not signature.output_fields: + return None def type_info(v): if v.annotation is not str: @@ -209,20 +226,7 @@ def format_assistant_message_content( return assistant_message_content def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: - sections = [(None, [])] - - for line in completion.splitlines(): - match = field_header_pattern.match(line.strip()) - if match: - # If the header pattern is found, split the rest of the line as content - header = match.group(1) - remaining_content = line[match.end() :].strip() - sections.append((header, [remaining_content] if remaining_content else [])) - else: - sections[-1][1].append(line) - - sections = [(k, "\n".join(v).strip()) for k, v in sections] - + sections = self._split_field_sections(completion) fields = {} for k, v in sections: if (k not in fields) and (k in signature.output_fields): @@ -245,6 +249,47 @@ def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: return fields + def parse_output_fields_to_parts( + self, + output: LMOutput, + signature: type[Signature], + ) -> dict[str, list[LMPart]]: + completion = output.text or "" + sections = self._split_field_sections(completion) + fields: dict[str, list[LMPart]] = {} + for key, value in sections: + if key not in fields and key in signature.output_fields: + fields[key] = [LMTextPart(text=value)] + if fields.keys() != signature.output_fields.keys(): + raise AdapterParseError( + adapter_name="ChatAdapter", + signature=signature, + lm_response=completion, + parsed_result={key: value[0].text if value else "" for key, value in fields.items()}, + ) + return fields + + def wrap_input_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + return [LMTextPart(text=f"[[ ## {field_name} ## ]]\n"), *parts] + + def wrap_output_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + return [LMTextPart(text=f"[[ ## {field_name} ## ]]\n"), *parts] + + def _split_field_sections(self, completion: str) -> list[tuple[str | None, str]]: + sections = [(None, [])] + + for line in completion.splitlines(): + match = field_header_pattern.match(line.strip()) + if match: + # If the header pattern is found, split the rest of the line as content + header = match.group(1) + remaining_content = line[match.end() :].strip() + sections.append((header, [remaining_content] if remaining_content else [])) + else: + sections[-1][1].append(line) + + return [(k, "\n".join(v).strip()) for k, v in sections] + def format_field_with_value(self, fields_with_values: dict[FieldInfoWithName, Any]) -> str: """ Formats the values of the specified fields according to the field's DSPy type (input or output), diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py index 59a3b6f563..6c4182c74e 100644 --- a/dspy/adapters/json_adapter.py +++ b/dspy/adapters/json_adapter.py @@ -7,6 +7,7 @@ import regex from pydantic.fields import FieldInfo +from dspy.adapters._type_runtime import _CallContext from dspy.adapters.chat_adapter import ChatAdapter, FieldInfoWithName from dspy.adapters.types.tool import ToolCalls from dspy.adapters.utils import ( @@ -17,6 +18,7 @@ translate_field_type, ) from dspy.clients.base_lm import BaseLM +from dspy.core.types import LMOutput, LMPart, LMTextPart from dspy.signatures.signature import Signature, SignatureMeta from dspy.utils.callback import BaseCallback from dspy.utils.exceptions import AdapterParseError @@ -38,9 +40,20 @@ def _has_open_ended_mapping(signature: SignatureMeta) -> bool: class JSONAdapter(ChatAdapter): - def __init__(self, callbacks: list[BaseCallback] | None = None, use_native_function_calling: bool = True): + def __init__( + self, + callbacks: list[BaseCallback] | None = None, + use_native_function_calling: bool = True, + native_response_types: list[type[type]] | None = None, + allow_parallel_tool_calls: bool | None = None, + ): # JSONAdapter uses native function calling by default. - super().__init__(callbacks=callbacks, use_native_function_calling=use_native_function_calling) + super().__init__( + callbacks=callbacks, + use_native_function_calling=use_native_function_calling, + native_response_types=native_response_types, + allow_parallel_tool_calls=allow_parallel_tool_calls, + ) def _json_adapter_call_common(self, lm, lm_kwargs, signature, demos, inputs, call_fn): """Common call logic to be used for both sync and async calls.""" @@ -68,10 +81,6 @@ def __call__( return result try: - structured_output_model = _get_structured_outputs_response_format( - signature, self.use_native_function_calling - ) - lm_kwargs["response_format"] = structured_output_model return super().__call__(lm, lm_kwargs, signature, demos, inputs) except Exception: logger.warning("Failed to use structured output format, falling back to JSON mode.") @@ -91,16 +100,38 @@ async def acall( return await result try: - structured_output_model = _get_structured_outputs_response_format( - signature, self.use_native_function_calling - ) - lm_kwargs["response_format"] = structured_output_model return await super().acall(lm, lm_kwargs, signature, demos, inputs) except Exception: logger.warning("Failed to use structured output format, falling back to JSON mode.") lm_kwargs["response_format"] = {"type": "json_object"} return await super().acall(lm, lm_kwargs, signature, demos, inputs) + def _prepare_request_kwargs( + self, + lm: BaseLM, + lm_kwargs: dict[str, Any], + source_signature: type[Signature], + render_signature: type[Signature], + context: _CallContext, + ) -> dict[str, Any]: + request_kwargs = dict(lm_kwargs) + if "response_format" in request_kwargs or "response_format" not in lm.supported_params: + return request_kwargs + + has_tool_calls = any(field.annotation == ToolCalls for field in source_signature.output_fields.values()) + if ( + _has_open_ended_mapping(render_signature) + or (not self.use_native_function_calling and has_tool_calls) + or not lm.supports_response_schema + ): + request_kwargs["response_format"] = {"type": "json_object"} + return request_kwargs + + request_kwargs["response_format"] = _get_structured_outputs_response_format( + render_signature, self.use_native_function_calling + ) + return request_kwargs + def format_field_structure(self, signature: type[Signature]) -> str: parts = [] parts.append("All interactions will be structured in the following way, with the appropriate values filled in.") @@ -116,11 +147,15 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo], role: parts.append("Inputs will have the following structure:") parts.append(format_signature_fields_for_instructions(signature.input_fields, role="user")) - parts.append("Outputs will be a JSON object with the following fields.") - parts.append(format_signature_fields_for_instructions(signature.output_fields, role="assistant")) + if signature.output_fields: + parts.append("Outputs will be a JSON object with the following fields.") + parts.append(format_signature_fields_for_instructions(signature.output_fields, role="assistant")) return "\n\n".join(parts).strip() - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: + if not signature.output_fields: + return None + def type_info(v): return ( f" (must be formatted as a valid Python {get_annotation_name(v.annotation)})" @@ -146,30 +181,32 @@ def format_assistant_message_content( return self.format_field_with_value(fields_with_values, role="assistant") def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: - fields = json_repair.loads(completion) + fields = self._parse_json_fields(completion, signature) + fields = {k: v for k, v in fields.items() if k in signature.output_fields} - if not isinstance(fields, dict): - pattern = r"\{(?:[^{}]|(?R))*\}" - match = regex.search(pattern, completion, regex.DOTALL) - if match: - completion = match.group(0) - fields = json_repair.loads(completion) + # Attempt to cast each value to type signature.output_fields[k].annotation. + for k, v in fields.items(): + if k in signature.output_fields: + fields[k] = parse_value(v, signature.output_fields[k].annotation) - if not isinstance(fields, dict): + if fields.keys() != signature.output_fields.keys(): raise AdapterParseError( adapter_name="JSONAdapter", signature=signature, lm_response=completion, - message="LM response cannot be serialized to a JSON object.", + parsed_result=fields, ) - fields = {k: v for k, v in fields.items() if k in signature.output_fields} - - # Attempt to cast each value to type signature.output_fields[k].annotation. - for k, v in fields.items(): - if k in signature.output_fields: - fields[k] = parse_value(v, signature.output_fields[k].annotation) + return fields + def parse_output_fields_to_parts( + self, + output: LMOutput, + signature: type[Signature], + ) -> dict[str, list[LMPart]]: + completion = output.text or "" + fields = self._parse_json_fields(completion, signature) + fields = {k: v for k, v in fields.items() if k in signature.output_fields} if fields.keys() != signature.output_fields.keys(): raise AdapterParseError( adapter_name="JSONAdapter", @@ -177,7 +214,28 @@ def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: lm_response=completion, parsed_result=fields, ) + return { + key: [LMTextPart(text=value if isinstance(value, str) else json.dumps(value, ensure_ascii=False))] + for key, value in fields.items() + } + + def _parse_json_fields(self, completion: str, signature: type[Signature]) -> dict[str, Any]: + fields = json_repair.loads(completion) + if not isinstance(fields, dict): + pattern = r"\{(?:[^{}]|(?R))*\}" + match = regex.search(pattern, completion, regex.DOTALL) + if match: + completion = match.group(0) + fields = json_repair.loads(completion) + + if not isinstance(fields, dict): + raise AdapterParseError( + adapter_name="JSONAdapter", + signature=signature, + lm_response=completion, + message="LM response cannot be serialized to a JSON object.", + ) return fields def format_field_with_value(self, fields_with_values: dict[FieldInfoWithName, Any], role: str = "user") -> str: diff --git a/dspy/adapters/two_step_adapter.py b/dspy/adapters/two_step_adapter.py index 0e427dea25..8908bb063f 100644 --- a/dspy/adapters/two_step_adapter.py +++ b/dspy/adapters/two_step_adapter.py @@ -1,12 +1,12 @@ from typing import Any -import json_repair - from dspy.adapters.base import Adapter from dspy.adapters.chat_adapter import ChatAdapter +from dspy.adapters._type_runtime import _CallContext from dspy.adapters.types import ToolCalls from dspy.adapters.utils import get_field_description_string from dspy.clients.base_lm import BaseLM +from dspy.core.types import LMMessage from dspy.signatures.field import InputField from dspy.signatures.signature import Signature, make_signature @@ -46,7 +46,15 @@ def __init__(self, extraction_model: BaseLM, **kwargs): self.extraction_model = extraction_model def format( - self, signature: type[Signature], demos: list[dict[str, Any]], inputs: dict[str, Any] + self, + signature: type[Signature], + demos: list[dict[str, Any]], + inputs: dict[str, Any], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + source_signature: type[Signature] | None = None, + prepared_messages: list[LMMessage] | None = None, ) -> list[dict[str, Any]]: """ Format a prompt for the first stage with the main LM. @@ -67,13 +75,44 @@ def format( task_description = self.format_task_description(signature) messages.append({"role": "system", "content": task_description}) - messages.extend(self.format_demos(signature, demos)) + messages.extend( + self.format_demos( + signature, + demos, + use_native_tool_calls=use_native_tool_calls, + context=context, + ) + ) + messages.extend(prepared_messages or []) # Format the current input messages.append({"role": "user", "content": self.format_user_message_content(signature, inputs)}) return messages + def render_messages( + self, + signature: type[Signature], + demos: list[dict[str, Any]], + inputs: dict[str, Any], + *, + use_native_tool_calls: bool = False, + context: _CallContext | None = None, + source_signature: type[Signature] | None = None, + prepared_messages: list[LMMessage] | None = None, + ) -> list[LMMessage]: + return self._coerce_lm_messages( + self.format( + signature, + demos, + inputs, + use_native_tool_calls=use_native_tool_calls, + context=context, + source_signature=source_signature, + prepared_messages=prepared_messages, + ) + ) + def parse(self, signature: Signature, completion: str) -> dict[str, Any]: """ Use a smaller LM (extraction_model) with chat adapter to extract structured data @@ -111,51 +150,47 @@ async def acall( demos: list[dict[str, Any]], inputs: dict[str, Any], ) -> list[dict[str, Any]]: - inputs = self.format(signature, demos, inputs) - - outputs = await lm.acall(messages=inputs, **lm_kwargs) - # The signature is supposed to be "text -> {original output fields}" - extractor_signature = self._create_extractor_signature(signature) + rendered = self._render_request(lm, lm_kwargs, signature, demos, inputs) + response = await self._acall_lm(lm, rendered.request) values = [] - tool_call_output_field_name = self._get_tool_call_output_field_name(signature) - for output in outputs: - output_logprobs = None - tool_calls = None - text = output - - if isinstance(output, dict): - text = output["text"] - output_logprobs = output.get("logprobs") - tool_calls = output.get("tool_calls") - - try: - # Call the smaller LM to extract structured data from the raw completion text with ChatAdapter - value = await ChatAdapter().acall( - lm=self.extraction_model, - lm_kwargs={}, - signature=extractor_signature, - demos=[], - inputs={"text": text}, + extractor_signature = self._create_extractor_signature(rendered.signature) + + for output in response.outputs: + if output.metadata.get("empty_legacy_outputs"): + continue + + has_text_output = bool(output.text and rendered.signature.output_fields) + has_tool_output = bool(output.tool_calls and tool_call_output_field_name) + + if has_text_output: + try: + value = await ChatAdapter().acall( + lm=self.extraction_model, + lm_kwargs={}, + signature=extractor_signature, + demos=[], + inputs={"text": output.text}, + ) + value = value[0] + for field_name in signature.output_fields: + value.setdefault(field_name, None) + except Exception as e: + raise ValueError(f"Failed to parse response from the original completion: {output}") from e + else: + value = dict.fromkeys(signature.output_fields.keys()) + + if has_tool_output: + value[tool_call_output_field_name] = ToolCalls.from_dict_list( + [{"name": call.name, "args": call.args, "id": call.id} for call in output.tool_calls] ) - value = value[0] - - except Exception as e: - raise ValueError(f"Failed to parse response from the original completion: {output}") from e - if tool_calls and tool_call_output_field_name: - tool_calls = [ - { - "name": v["function"]["name"], - "args": json_repair.loads(v["function"]["arguments"]), - } - for v in tool_calls - ] - value[tool_call_output_field_name] = ToolCalls.from_dict_list(tool_calls) + if not (has_text_output or has_tool_output): + raise ValueError(f"Failed to parse response from the original completion: {output}") - if output_logprobs is not None: - value["logprobs"] = output_logprobs + if output.logprobs is not None: + value["logprobs"] = output.logprobs values.append(value) return values @@ -180,6 +215,7 @@ def format_user_message_content( inputs: dict[str, Any], prefix: str = "", suffix: str = "", + main_request: bool = False, ) -> str: parts = [prefix] diff --git a/dspy/adapters/types/__init__.py b/dspy/adapters/types/__init__.py index 5ec8043021..c70878dc2e 100644 --- a/dspy/adapters/types/__init__.py +++ b/dspy/adapters/types/__init__.py @@ -5,6 +5,17 @@ from dspy.adapters.types.history import History from dspy.adapters.types.image import Image from dspy.adapters.types.reasoning import Reasoning -from dspy.adapters.types.tool import Tool, ToolCalls +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls -__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "Code", "Reasoning"] +__all__ = [ + "History", + "Image", + "Audio", + "File", + "Type", + "Tool", + "ToolCalls", + "ToolCallResults", + "Code", + "Reasoning", +] diff --git a/dspy/adapters/types/base_type.py b/dspy/adapters/types/base_type.py index 13a55727f2..2a629559b4 100644 --- a/dspy/adapters/types/base_type.py +++ b/dspy/adapters/types/base_type.py @@ -5,13 +5,12 @@ import json_repair import pydantic -from dspy.clients.base_lm import BaseLM +from dspy.adapters._type_runtime import _TypeParseContext, _TypeRenderContext +from dspy.core.types import LMPart if TYPE_CHECKING: from litellm import ModelResponseStream - from dspy.signatures.signature import Signature - CUSTOM_TYPE_START_IDENTIFIER = "<>" CUSTOM_TYPE_END_IDENTIFIER = "<>" @@ -36,6 +35,16 @@ def format(self) -> list[dict[str, Any]]: def format(self) -> list[dict[str, Any]] | str: raise NotImplementedError + def default_render_input(self, ctx: _TypeRenderContext) -> list[LMPart]: + return ctx.adapter.value_to_lm_parts(self, ctx.field_info) + + def default_render_output(self, ctx: _TypeRenderContext) -> list[LMPart]: + return ctx.adapter.value_to_lm_parts(self, ctx.field_info) + + @classmethod + def default_parse_output(cls, ctx: _TypeParseContext) -> object: + return ctx.adapter.lm_parts_to_value(ctx.parts, ctx.field_info) + @classmethod def description(cls) -> str: """Description of the custom type""" @@ -76,31 +85,6 @@ def serialize_model(self): ) return formatted - @classmethod - def adapt_to_native_lm_feature( - cls, - signature: type["Signature"], - field_name: str, - lm: BaseLM, - lm_kwargs: dict[str, Any], - ) -> type["Signature"]: - """Adapt the custom type to the native LM feature if possible. - - When the LM and configuration supports the related native LM feature, e.g., native tool calling, native - reasoning, etc., we adapt the signature and `lm_kwargs` to enable the native LM feature. - - Args: - signature: The DSPy signature for the LM call. - field_name: The name of the field in the signature to adapt to the native LM feature. - lm: The LM instance. - lm_kwargs: The keyword arguments for the LM call, subject to in-place updates if adaptation if required. - - Returns: - The adapted signature. If the custom type is not natively supported by the LM, return the original - signature. - """ - return signature - @classmethod def is_streamable(cls) -> bool: """Whether the custom type is streamable.""" @@ -166,8 +150,11 @@ def split_message_content_for_custom_types(messages: list[dict[str, Any]]) -> li pattern = rf"{CUSTOM_TYPE_START_IDENTIFIER}(.*?){CUSTOM_TYPE_END_IDENTIFIER}" result = [] last_end = 0 - # DSPy adapter always formats user input into a string content before custom type splitting - content: str = message["content"] + # Normalized adapter paths may have already rendered custom values into + # content blocks. The legacy marker splitter only applies to text. + content = message["content"] + if not isinstance(content, str): + continue for match in re.finditer(pattern, content, re.DOTALL): start, end = match.span() diff --git a/dspy/adapters/types/citation.py b/dspy/adapters/types/citation.py index a8c5461a58..b6640968cb 100644 --- a/dspy/adapters/types/citation.py +++ b/dspy/adapters/types/citation.py @@ -167,12 +167,6 @@ def __getitem__(self, index): """Allow indexing into citations.""" return self.citations[index] - @classmethod - def adapt_to_native_lm_feature(cls, signature, field_name, lm, lm_kwargs) -> bool: - if lm.model.startswith("anthropic/"): - return signature.delete(field_name) - return signature - @classmethod def is_streamable(cls) -> bool: """Whether the Citations type is streamable.""" diff --git a/dspy/adapters/types/history.py b/dspy/adapters/types/history.py index 6dda4f9b7c..1c5cbd3360 100644 --- a/dspy/adapters/types/history.py +++ b/dspy/adapters/types/history.py @@ -1,68 +1,102 @@ -from typing import Any +from typing import Any, Callable import pydantic +from pydantic import Field, model_validator +from dspy.adapters.types.tool import ToolCallResults, ToolCalls -class History(pydantic.BaseModel): - """Class representing the conversation history. - - The conversation history is a list of messages, each message entity should have keys from the associated signature. - For example, if you have the following signature: - - ``` - class MySignature(dspy.Signature): - question: str = dspy.InputField() - history: dspy.History = dspy.InputField() - answer: str = dspy.OutputField() - ``` - - Then the history should be a list of dictionaries with keys "question" and "answer". - - Examples: - ``` - import dspy - - dspy.configure(lm=dspy.LM("openai/gpt-4o-mini")) - - class MySignature(dspy.Signature): - question: str = dspy.InputField() - history: dspy.History = dspy.InputField() - answer: str = dspy.OutputField() - history = dspy.History( - messages=[ - {"question": "What is the capital of France?", "answer": "Paris"}, - {"question": "What is the capital of Germany?", "answer": "Berlin"}, - ] - ) - - predict = dspy.Predict(MySignature) - outputs = predict(question="What is the capital of France?", history=history) - ``` - - Example of capturing the conversation history: - ``` - import dspy - - dspy.configure(lm=dspy.LM("openai/gpt-4o-mini")) - - class MySignature(dspy.Signature): - question: str = dspy.InputField() - history: dspy.History = dspy.InputField() - answer: str = dspy.OutputField() - - predict = dspy.Predict(MySignature) - outputs = predict(question="What is the capital of France?") - history = dspy.History(messages=[{"question": "What is the capital of France?", **outputs}]) - outputs_with_history = predict(question="Are you sure?", history=history) - ``` - """ +class History(pydantic.BaseModel): + """Reusable ordered field history.""" - messages: list[dict[str, Any]] + messages: list[dict[str, Any]] = Field(default_factory=list) model_config = pydantic.ConfigDict( - frozen=True, str_strip_whitespace=True, - validate_assignment=True, extra="forbid", ) + + @model_validator(mode="after") + def _normalize_messages(self) -> "History": + self.messages = [ + self._normalize_message(message) if isinstance(message, dict) else message for message in self.messages + ] + return self + + def __init__(self, *args: Any, compact_fn: Callable[["History"], None] | None = None, **kwargs: Any): + super().__init__(*args, **kwargs) + object.__setattr__(self, "_compact_fn", compact_fn) + + @pydantic.model_serializer() + def serialize_model(self) -> dict[str, Any]: + return {"messages": [self._serialize_message(message) for message in self.messages]} + + def compact_if_needed(self) -> None: + fn = getattr(self, "_compact_fn", None) + if fn is not None: + fn(self) + + def append(self, message: dict[str, Any]) -> dict[str, Any]: + message = self._normalize_message(dict(message)) + self.messages.append(message) + return message + + def to_lm_messages(self, adapter: Any, signature: type[Any], *, use_native_tool_calls: bool = False) -> list[Any]: + return adapter.format_history(self, signature, use_native_tool_calls=use_native_tool_calls) + + @staticmethod + def _normalize_message(message: dict[str, Any]) -> dict[str, Any]: + normalized = {} + for key, value in message.items(): + if key == "tool_calls" and isinstance(value, list): + normalized[key] = ToolCalls.model_validate(value) + elif key == "tool_call_results" and isinstance(value, list): + normalized[key] = ToolCallResults.model_validate(value) + elif isinstance(value, dict) and set(value.keys()) == {"tool_calls"} and isinstance(value["tool_calls"], list): + normalized[key] = ToolCalls.model_validate(value) + elif ( + isinstance(value, dict) + and set(value.keys()) == {"tool_call_results"} + and isinstance(value["tool_call_results"], list) + ): + normalized[key] = ToolCallResults.model_validate(value) + else: + normalized[key] = value + return normalized + + @staticmethod + def _serialize_message(message: dict[str, Any]) -> dict[str, Any]: + serialized = {} + for key, value in message.items(): + if isinstance(value, (ToolCalls, ToolCallResults)): + serialized[key] = value.format() + elif hasattr(value, "model_dump"): + serialized[key] = value.model_dump() + else: + serialized[key] = value + return serialized + + +def truncate_oldest_actions(history: History, *, max_tokens: int = 200_000, keep_n: int = 3) -> None: + if len(str(history.messages)) // 4 <= max_tokens: + return + + action_indices = [ + idx + for idx, message in enumerate(history.messages) + if isinstance(message, dict) and any(isinstance(value, ToolCalls) for value in message.values()) + ] + drop_count = len(action_indices) - keep_n + if drop_count <= 0: + return + + drop_indices = set(action_indices[:drop_count]) + + history.messages[:] = [message for idx, message in enumerate(history.messages) if idx not in drop_indices] + + +def make_truncate_oldest_actions(max_tokens: int = 200_000, keep_n: int = 3) -> Callable[[History], None]: + def _compact(history: History) -> None: + truncate_oldest_actions(history, max_tokens=max_tokens, keep_n=keep_n) + + return _compact diff --git a/dspy/adapters/types/reasoning.py b/dspy/adapters/types/reasoning.py index c4fa3de742..08f229348f 100644 --- a/dspy/adapters/types/reasoning.py +++ b/dspy/adapters/types/reasoning.py @@ -1,12 +1,8 @@ -from typing import TYPE_CHECKING, Any, Optional +from typing import Any, Optional import pydantic from dspy.adapters.types.base_type import Type -from dspy.clients.base_lm import BaseLM - -if TYPE_CHECKING: - from dspy.signatures.signature import Signature class Reasoning(Type): @@ -42,40 +38,6 @@ def validate_input(cls, data: Any): raise ValueError(f"Received invalid value for `dspy.Reasoning`: {data}") - @classmethod - def adapt_to_native_lm_feature( - cls, - signature: type["Signature"], - field_name: str, - lm: BaseLM, - lm_kwargs: dict[str, Any], - ) -> type["Signature"]: - if "reasoning_effort" in lm_kwargs: - # `lm_kwargs` overrides `lm.kwargs`. - reasoning_effort = lm_kwargs["reasoning_effort"] - elif "reasoning_effort" in lm.kwargs: - reasoning_effort = lm.kwargs["reasoning_effort"] - else: - # Turn on the native reasoning explicitly if Reasoning field is present in the signature and no explicit - # reasoning effort is set in `lm_kwargs` or `lm.kwargs`. - reasoning_effort = "low" - - if reasoning_effort is None or not lm.supports_reasoning: - # If users explicitly set `reasoning_effort` to None or the LM doesn't support reasoning, we don't enable - # native reasoning. - return signature - - if "gpt-5" in lm.model and lm.model_type == "chat": - # There is a caveat of Litellm as 1.79.0 that when using the chat completion API on GPT-5 family models, - # the reasoning content is not available in the response. As a workaround, we don't enable the native - # reasoning feature for GPT-5 family models when using the chat completion API. - # Litellm issue: https://github.com/BerriAI/litellm/issues/14748 - return signature - - lm_kwargs["reasoning_effort"] = reasoning_effort - # Delete the reasoning field from the signature to use the native reasoning feature. - return signature.delete(field_name) - @classmethod def parse_lm_response(cls, response: str | dict[str, Any]) -> Optional["Reasoning"]: """Parse the LM response into a Reasoning object.""" diff --git a/dspy/adapters/types/tool.py b/dspy/adapters/types/tool.py index e6deb9b7c2..4ceea4e2c7 100644 --- a/dspy/adapters/types/tool.py +++ b/dspy/adapters/types/tool.py @@ -1,12 +1,13 @@ import asyncio import inspect -from typing import TYPE_CHECKING, Any, Callable, get_origin, get_type_hints +from typing import TYPE_CHECKING, Any, Callable, ClassVar, get_origin, get_type_hints import pydantic from jsonschema import ValidationError, validate from pydantic import BaseModel, TypeAdapter, create_model from dspy.adapters.types.base_type import Type +from dspy.core.types import LMMessage, LMTextPart, LMToolCallPart, LMToolResultPart, LMToolSpec from dspy.dsp.utils.settings import settings from dspy.utils.callback import with_callbacks @@ -162,6 +163,14 @@ def format_as_litellm_function_call(self): }, } + def to_lm_tool_spec(self) -> LMToolSpec: + args = self.args or {} + return LMToolSpec( + name=self.name or "", + description=self.desc, + parameters={"type": "object", "properties": args, "required": list(args.keys())}, + ) + def _run_async_in_sync(self, coroutine): try: loop = asyncio.get_running_loop() @@ -260,18 +269,21 @@ def __str__(self): class ToolCalls(Type): + JSON_SCHEMA_EXTRA_MAX_ITEMS_KEY: ClassVar[str] = "__dspy_tool_calls_max_items" + class ToolCall(Type): name: str args: dict[str, Any] + id: str | None = None def format(self): - return { - "type": "function", - "function": { - "name": self.name, - "arguments": self.args, - }, - } + formatted = {"name": self.name, "args": self.args} + if self.id is not None: + formatted["id"] = self.id + return formatted + + def to_lm_part(self, tool_call_id: str | None = None) -> LMToolCallPart: + return LMToolCallPart(id=tool_call_id or self.id, name=self.name, args=self.args) def execute(self, functions: dict[str, Any] | list[Tool] | None = None) -> Any: """Execute this individual tool call and return its result. @@ -320,6 +332,41 @@ def execute(self, functions: dict[str, Any] | list[Tool] | None = None) -> Any: tool_calls: list[ToolCall] + @classmethod + def json_schema(cls, *, max_items: int | None = None) -> dict[str, Any]: + """Return the prompt-facing JSON schema for ToolCalls.""" + if max_items is not None and max_items < 1: + raise ValueError("ToolCalls JSON schema `max_items` must be at least 1.") + + schema = pydantic.TypeAdapter(cls).json_schema() + if max_items is not None: + schema.setdefault("properties", {}).setdefault("tool_calls", {})["maxItems"] = max_items + return schema + + @classmethod + def json_schema_extra_for_max_items(cls, max_items: int | None) -> dict[str, Any]: + if max_items is None: + return {} + return {cls.JSON_SCHEMA_EXTRA_MAX_ITEMS_KEY: max_items} + + @classmethod + def max_items_from_field_info(cls, field_info: Any) -> int | None: + if field_info is None: + return None + field_extra = getattr(field_info, "json_schema_extra", None) or {} + return field_extra.get(cls.JSON_SCHEMA_EXTRA_MAX_ITEMS_KEY) + + def validate_max_items(self, max_items: int | None) -> None: + if max_items is None: + return + if len(self.tool_calls) <= max_items: + return + raise ValueError( + f"`dspy.ToolCalls` received {len(self.tool_calls)} tool calls, but this call permits at most " + f"{max_items}. Set `allow_parallel_tool_calls=True` to accept multiple tool calls in one LM turn, " + "or return one tool call per turn." + ) + @classmethod def from_dict_list(cls, tool_calls_dicts: list[dict[str, Any]]) -> "ToolCalls": """Convert a list of dictionaries to a ToolCalls instance. @@ -340,8 +387,7 @@ def from_dict_list(cls, tool_calls_dicts: list[dict[str, Any]]) -> "ToolCalls": tool_calls = ToolCalls.from_dict_list(tool_calls_dict) ``` """ - tool_calls = [cls.ToolCall(**item) for item in tool_calls_dicts] - return cls(tool_calls=tool_calls) + return cls.model_validate(tool_calls_dicts) @classmethod def description(cls) -> str: @@ -350,41 +396,162 @@ def description(cls) -> str: "Arguments must be provided in JSON format." ) - def format(self) -> list[dict[str, Any]]: - # The tool_call field is compatible with OpenAI's tool calls schema. + def format(self) -> dict[str, Any]: return { "tool_calls": [tool_call.format() for tool_call in self.tool_calls], } + @classmethod + def parse_lm_response(cls, response: str | dict[str, Any]) -> "ToolCalls | None": + if not isinstance(response, dict): + return None + tool_calls = response.get("tool_calls") + if not tool_calls: + return None + return cls.model_validate(tool_calls) + + def to_lm_parts(self, id_prefix: str | None = None) -> list[LMToolCallPart]: + return [ + tool_call.to_lm_part(f"{id_prefix}_{idx}" if id_prefix is not None and tool_call.id is None else None) + for idx, tool_call in enumerate(self.tool_calls) + ] + + def with_call_ids(self, id_prefix: str) -> "ToolCalls": + tool_calls = [ + tool_call if tool_call.id is not None else tool_call.model_copy(update={"id": f"{id_prefix}_{idx}"}) + for idx, tool_call in enumerate(self.tool_calls) + ] + return self.model_copy(update={"tool_calls": tool_calls}) + + @classmethod + def _canonical_tool_call(cls, item: Any) -> Any: + if isinstance(item, cls.ToolCall): + return item.model_dump() + if isinstance(item, dict): + if "name" in item and "args" in item: + normalized = {"name": item["name"], "args": item["args"]} + if "id" in item: + normalized["id"] = item["id"] + return normalized + return item + if hasattr(item, "name") and hasattr(item, "args"): + normalized = {"name": item.name, "args": item.args} + tool_call_id = getattr(item, "id", None) + if tool_call_id is not None: + normalized["id"] = tool_call_id + return normalized + return item + @pydantic.model_validator(mode="before") @classmethod def validate_input(cls, data: Any): if isinstance(data, cls): return data - # Handle case where data is a list of dicts with "name" and "args" keys - if isinstance(data, list) and all( - isinstance(item, dict) and "name" in item and "args" in item for item in data - ): - return {"tool_calls": [cls.ToolCall(**item) for item in data]} - # Handle case where data is a dict + tool_calls_data = None + if isinstance(data, list): + tool_calls_data = data elif isinstance(data, dict): if "tool_calls" in data: - # Handle case where data is a dict with "tool_calls" key tool_calls_data = data["tool_calls"] - if isinstance(tool_calls_data, list): - return { - "tool_calls": [ - cls.ToolCall(**item) if isinstance(item, dict) else item for item in tool_calls_data - ] - } elif "name" in data and "args" in data: - # Handle case where data is a dict with "name" and "args" keys - return {"tool_calls": [cls.ToolCall(**data)]} + return {"tool_calls": [cls._canonical_tool_call(data)]} + + if isinstance(tool_calls_data, list): + normalized = [cls._canonical_tool_call(item) for item in tool_calls_data] + if all(isinstance(item, dict) and "name" in item and "args" in item for item in normalized): + return {"tool_calls": normalized} raise ValueError(f"Received invalid value for `dspy.ToolCalls`: {data}") +class _ToolCallResult(BaseModel): + call_id: str | None = None + name: str | None = None + value: Any + is_error: bool = False + + @staticmethod + def _format_value(value: Any) -> str: + if isinstance(value, list): + return "\n".join(str(item) for item in value) + return str(value) + + def to_lm_message(self) -> LMMessage: + return LMMessage( + role="tool", + parts=[ + LMToolResultPart( + call_id=self.call_id, + name=self.name, + content=[LMTextPart(text=self._format_value(self.value))], + is_error=self.is_error, + ) + ], + ) + + def format(self) -> dict[str, Any]: + formatted = {"value": self.value, "is_error": self.is_error} + if self.call_id is not None: + formatted["call_id"] = self.call_id + if self.name is not None: + formatted["name"] = self.name + return formatted + + +class ToolCallResults(Type): + tool_call_results: list[_ToolCallResult] + + @classmethod + def from_dict_list(cls, tool_call_result_dicts: list[dict[str, Any]]) -> "ToolCallResults": + return cls.model_validate(tool_call_result_dicts) + + @classmethod + def from_tool_calls_and_values( + cls, + tool_calls: list[ToolCalls.ToolCall], + values: list[Any], + is_errors: list[bool], + ) -> "ToolCallResults": + return cls( + tool_call_results=[ + _ToolCallResult( + call_id=tool_call.id, + name=tool_call.name, + value=value, + is_error=is_error, + ) + for tool_call, value, is_error in zip(tool_calls, values, is_errors, strict=True) + ] + ) + + @classmethod + def description(cls) -> str: + return ( + "Tool call results, including the tool call id, tool name, returned value, and whether the tool " + "execution failed." + ) + + def format(self) -> dict[str, Any]: + return {"tool_call_results": [result.format() for result in self.tool_call_results]} + + def to_lm_messages(self) -> list[LMMessage]: + return [result.to_lm_message() for result in self.tool_call_results] + + @pydantic.model_validator(mode="before") + @classmethod + def validate_input(cls, data: Any): + if isinstance(data, cls): + return data + if isinstance(data, list): + return {"tool_call_results": data} + if isinstance(data, dict) and "tool_call_results" in data: + return data + if isinstance(data, dict) and "value" in data: + return {"tool_call_results": [data]} + raise ValueError(f"Received invalid value for `dspy.ToolCallResults`: {data}") + + def _resolve_json_schema_reference(schema: dict) -> dict: """Recursively resolve json model schema, expanding all references.""" diff --git a/dspy/adapters/utils.py b/dspy/adapters/utils.py index b45eda5593..2dd1f0a62e 100644 --- a/dspy/adapters/utils.py +++ b/dspy/adapters/utils.py @@ -14,6 +14,7 @@ from dspy.adapters.types.base_type import Type as DspyType from dspy.adapters.types.code import Code from dspy.adapters.types.reasoning import Reasoning +from dspy.adapters.types.tool import ToolCalls from dspy.signatures.utils import get_dspy_field_type @@ -74,7 +75,7 @@ def format_field_value(field_info: FieldInfo, value: Any, assume_text=True) -> s return {"type": "text", "text": string_value} -def _get_json_schema(field_type): +def _get_json_schema(field_type, field_info=None): def move_type_to_front(d): # Move the 'type' key to the front of the dictionary, recursively, for LLM readability/adherence. if isinstance(d, Mapping): @@ -85,7 +86,10 @@ def move_type_to_front(d): return [move_type_to_front(item) for item in d] return d - schema = pydantic.TypeAdapter(field_type).json_schema() + if field_type is ToolCalls: + schema = ToolCalls.json_schema(max_items=ToolCalls.max_items_from_field_info(field_info)) + else: + schema = pydantic.TypeAdapter(field_type).json_schema() schema = move_type_to_front(schema) return schema @@ -112,7 +116,8 @@ def translate_field_type(field_name, field_info): # Code has a rich type description already; avoid duplicating its large schema block. desc = "" else: - desc = f"must adhere to the JSON schema: {json.dumps(_get_json_schema(field_type), ensure_ascii=False)}" + json_schema = json.dumps(_get_json_schema(field_type, field_info), ensure_ascii=False) + desc = f"must adhere to the JSON schema: {json_schema}" desc = (" " * 8) + f"# note: the value you produce {desc}" if desc else "" return f"{{{field_name}}}{desc}" diff --git a/dspy/adapters/xml_adapter.py b/dspy/adapters/xml_adapter.py index 662b1d2868..d835c04c8d 100644 --- a/dspy/adapters/xml_adapter.py +++ b/dspy/adapters/xml_adapter.py @@ -5,6 +5,7 @@ from dspy.adapters.chat_adapter import ChatAdapter, FieldInfoWithName from dspy.adapters.utils import format_field_value, translate_field_type +from dspy.core.types import LMOutput, LMPart, LMTextPart from dspy.signatures.signature import Signature from dspy.utils.callback import BaseCallback @@ -38,7 +39,8 @@ def format_signature_fields_for_instructions(fields: dict[str, FieldInfo]): ) parts.append(format_signature_fields_for_instructions(signature.input_fields)) - parts.append(format_signature_fields_for_instructions(signature.output_fields)) + if signature.output_fields: + parts.append(format_signature_fields_for_instructions(signature.output_fields)) return "\n\n".join(parts).strip() def format_user_message_content( @@ -79,19 +81,17 @@ def format_assistant_message_content( }, ) - def user_message_output_requirements(self, signature: type[Signature]) -> str: + def user_message_output_requirements(self, signature: type[Signature]) -> str | None: + if not signature.output_fields: + return None + message = "Respond with the corresponding output fields wrapped in XML tags " message += ", then ".join(f"`<{f}>`" for f in signature.output_fields) message += "." return message def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: - fields = {} - for match in self.field_pattern.finditer(completion): - name = match.group("name") - content = match.group("content").strip() - if name in signature.output_fields and name not in fields: - fields[name] = content + fields = self._parse_xml_fields(completion, signature) # Cast values using base class parse_value helper for k, v in fields.items(): fields[k] = self._parse_field_value(signature.output_fields[k], v, completion, signature) @@ -106,6 +106,39 @@ def parse(self, signature: type[Signature], completion: str) -> dict[str, Any]: ) return fields + def parse_output_fields_to_parts( + self, + output: LMOutput, + signature: type[Signature], + ) -> dict[str, list[LMPart]]: + completion = output.text or "" + fields = self._parse_xml_fields(completion, signature) + if fields.keys() != signature.output_fields.keys(): + from dspy.utils.exceptions import AdapterParseError + + raise AdapterParseError( + adapter_name="XMLAdapter", + signature=signature, + lm_response=completion, + parsed_result=fields, + ) + return {key: [LMTextPart(text=value)] for key, value in fields.items()} + + def wrap_input_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + return [LMTextPart(text=f"<{field_name}>\n"), *parts, LMTextPart(text=f"\n")] + + def wrap_output_field_parts(self, field_name: str, parts: list[LMPart]) -> list[LMPart]: + return [LMTextPart(text=f"<{field_name}>\n"), *parts, LMTextPart(text=f"\n")] + + def _parse_xml_fields(self, completion: str, signature: type[Signature]) -> dict[str, str]: + fields = {} + for match in self.field_pattern.finditer(completion): + name = match.group("name") + content = match.group("content").strip() + if name in signature.output_fields and name not in fields: + fields[name] = content + return fields + def _parse_field_value(self, field_info, raw, completion, signature): from dspy.adapters.utils import parse_value diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py index c371f3e06b..be5240e20f 100644 --- a/dspy/clients/__init__.py +++ b/dspy/clients/__init__.py @@ -4,7 +4,7 @@ from typing import Any from dspy.clients._litellm import get_litellm -from dspy.clients.base_lm import BaseLM, inspect_history +from dspy.clients.base_lm import BaseLM, LMCapabilities, inspect_history from dspy.clients.cache import Cache from dspy.clients.embedding import Embedder from dspy.clients.lm import LM @@ -39,7 +39,7 @@ def configure_cache( safe_types: Additional types to allow when restrict_pickle is True. """ - DSPY_CACHE = Cache( + dspy_cache = Cache( enable_disk_cache, enable_memory_cache, disk_cache_dir, @@ -52,7 +52,7 @@ def configure_cache( import dspy # Update the reference to point to the new cache - dspy.cache = DSPY_CACHE + dspy.cache = dspy_cache @@ -112,6 +112,7 @@ def disable_litellm_logging(): __all__ = [ "BaseLM", + "LMCapabilities", "LM", "Provider", "TrainingJob", diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.py index 760a2679b5..2acb49e3b7 100644 --- a/dspy/clients/base_lm.py +++ b/dspy/clients/base_lm.py @@ -1,7 +1,11 @@ +import copy as copy_module import datetime import uuid +from dataclasses import dataclass, field from typing import Any, TextIO +from typing_extensions import Self + from dspy.dsp.utils import settings from dspy.utils.callback import with_callbacks from dspy.utils.inspect_history import pretty_print_history @@ -10,6 +14,27 @@ GLOBAL_HISTORY = [] +@dataclass(frozen=True) +class LMCapabilities: + """Optional model and deployment metadata for an LM backend. + + Capabilities are descriptive hints. Adapters can use them to select native + paths, but concrete LM implementations still decide how to handle requests. + """ + + function_calling: bool = False + reasoning: bool = False + response_schema: bool = False + streaming: bool = False + input_image: bool = False + input_audio: bool = False + input_file: bool = False + output_image: bool = False + output_audio: bool = False + tool_results: bool = False + extensions: dict[str, Any] = field(default_factory=dict) + + class BaseLM: """Base class for handling LLM calls. @@ -60,32 +85,55 @@ def forward(self, prompt, messages=None, **kwargs): ``` """ - def __init__(self, model, model_type="chat", temperature=0.0, max_tokens=1000, cache=True, **kwargs): + def __init__( + self, + model, + model_type="chat", + temperature=0.0, + max_tokens=1000, + cache=True, + callbacks=None, + num_retries=0, + **kwargs, + ): self.model = model self.model_type = model_type self.cache = cache + self.callbacks = callbacks or [] + self.num_retries = num_retries self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs) self.history = [] + self._warned_zero_temp_rollout = False + + @property + def capabilities(self) -> LMCapabilities: + """Native metadata available for this model instance.""" + return self.get_capabilities() + + def get_capabilities(self) -> LMCapabilities: + """Return optional native model and deployment hints.""" + return LMCapabilities() @property def supports_function_calling(self) -> bool: """Whether the model supports function calling (tool use).""" - return False + return self.capabilities.function_calling @property def supports_reasoning(self) -> bool: """Whether the model supports native reasoning (extended thinking).""" - return False + return self.capabilities.reasoning @property def supports_response_schema(self) -> bool: """Whether the model supports structured output via response schema.""" - return False + return self.capabilities.response_schema @property def supported_params(self) -> set[str]: """Set of supported OpenAI-style parameter names for the model.""" - return set() + supported = self.capabilities.extensions.get("supported_params", set()) + return set(supported) if supported else set() def _process_lm_response(self, response, prompt, messages, **kwargs): merged_kwargs = {**self.kwargs, **kwargs} @@ -190,23 +238,43 @@ async def aforward( """ raise NotImplementedError("Subclasses must implement this method.") + def dump_state(self) -> dict[str, Any]: + """Return a sanitized reconstruction state for this LM.""" + filtered_kwargs = {key: value for key, value in self.kwargs.items() if key != "api_key"} + return { + "model": self.model, + "model_type": self.model_type, + "cache": self.cache, + "num_retries": self.num_retries, + **filtered_kwargs, + } + + @classmethod + def load_state(cls, state: dict[str, Any]) -> Self: + """Reconstruct this LM from `dump_state()` output.""" + return cls(**state) + def copy(self, **kwargs): """Returns a copy of the language model with possibly updated parameters. Any provided keyword arguments update the corresponding attributes or LM kwargs of the copy. For example, ``lm.copy(rollout_id=1, temperature=1.0)`` returns an LM whose requests use a different rollout ID at non-zero temperature to bypass cache collisions. - """ - import copy + The default implementation uses a shallow runtime copy so provider clients, sessions, + and local model handles are preserved by reference while DSPy-owned mutable state is + isolated on the copy. + """ - new_instance = copy.deepcopy(self) + new_instance = copy_module.copy(self) new_instance.history = [] + new_instance.callbacks = list(getattr(self, "callbacks", []) or []) + new_instance.kwargs = dict(getattr(self, "kwargs", {}) or {}) for key, value in kwargs.items(): - if hasattr(self, key): + if hasattr(new_instance, key): setattr(new_instance, key, value) - if (key in self.kwargs) or (not hasattr(self, key)): + if (key in new_instance.kwargs) or (not hasattr(self, key)): if value is None: new_instance.kwargs.pop(key, None) else: diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py index 51ad677e0e..1405a745c9 100644 --- a/dspy/clients/lm.py +++ b/dspy/clients/lm.py @@ -20,7 +20,7 @@ from dspy.utils.callback import BaseCallback from dspy.utils.exceptions import ContextWindowExceededError -from .base_lm import BaseLM +from .base_lm import BaseLM, LMCapabilities logger = logging.getLogger(__name__) @@ -123,6 +123,16 @@ def _provider_name(self) -> str: return self.model.split("/", 1)[0] return "openai" + def get_capabilities(self) -> LMCapabilities: + params = self.supported_params + return LMCapabilities( + function_calling=self.supports_function_calling, + reasoning=self.supports_reasoning, + response_schema=self.supports_response_schema, + streaming=True, + extensions={"supported_params": params}, + ) + @property def supports_function_calling(self) -> bool: return _get_litellm().supports_function_calling(model=self.model) diff --git a/dspy/clients/openai_format.py b/dspy/clients/openai_format.py new file mode 100644 index 0000000000..4fd88ece5b --- /dev/null +++ b/dspy/clients/openai_format.py @@ -0,0 +1,903 @@ +"""Translate between DSPy's LM types and OpenAI-shaped JSON. + +This module does not call any provider. It only maps data shapes: + +```text +LMRequest -> OpenAI Chat / Responses / text-completion kwargs +provider response -> LMResponse +``` + +The concrete OpenAI backends import these functions, then add transport, +streaming, and caching around them. Read the file in +this order when learning it: + +1. `to_openai_chat_request()` maps chat-completion requests. +2. `to_openai_responses_request()` maps Responses API requests. +3. `to_openai_text_request()` maps legacy text-completion requests. +4. `completion_to_lm_response()` and `responses_to_lm_response()` map outputs. +5. The final utility section handles media sources, data URIs, and object/dict + access. +""" + +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +from typing import Any + +import pydantic + +from dspy.core.types import ( + LMAudioPart, + LMBinaryPart, + LMCitationPart, + LMConfig, + LMDocumentPart, + LMImagePart, + LMMessage, + LMOutput, + LMRefusalPart, + LMRequest, + LMResponse, + LMTextPart, + LMThinkingPart, + LMToolCallPart, + LMToolChoice, + LMToolResultPart, + LMToolSpec, + LMUsage, + LMVideoPart, +) + +__all__ = [ + "to_openai_chat_request", + "to_openai_responses_request", + "to_openai_text_request", + "completion_to_lm_response", + "responses_to_lm_response", + "provider_tool_call_to_part", + "responses_function_call_to_part", + "usage_from_response", +] + + +# --------------------------------------------------------------------------- +# DSPy request -> OpenAI Chat Completions +# +# Chat Completions expects: +# {"model": ..., "messages": [...], "tools": [...], ...config} +# Each DSPy message stays one OpenAI message, except assistant tool calls are +# split into OpenAI's top-level `tool_calls` field on the assistant message. +# --------------------------------------------------------------------------- + + +def to_openai_chat_request(request: LMRequest) -> dict[str, Any]: + """Convert a normalized DSPy request into Chat Completions kwargs.""" + data = {"model": request.model, "messages": [message_to_openai_chat(message) for message in request.messages]} + data.update(common_config_kwargs(request.config, model=request.model, endpoint="chat")) + if request.config.tool_choice is not None: + data.update(tool_choice_to_openai(request.config.tool_choice)) + if request.tools: + data["tools"] = [tool_to_openai(tool) for tool in request.tools] + return data + + +def message_to_openai_chat(message: LMMessage) -> dict[str, Any]: + """Convert one DSPy message into one Chat Completions message.""" + output: dict[str, Any] = {"role": message.role} + if message.name is not None: + output["name"] = message.name + + if message.role == "assistant": + tool_calls = [part for part in message.parts if isinstance(part, LMToolCallPart)] + content_parts = [part for part in message.parts if not isinstance(part, LMToolCallPart)] + output["content"] = parts_to_openai_content(content_parts) + if tool_calls: + output["tool_calls"] = [assistant_tool_call_to_openai(part) for part in tool_calls] + return output + + if message.role == "tool" and len(message.parts) == 1 and isinstance(message.parts[0], LMToolResultPart): + result = message.parts[0] + output.update(tool_result_to_openai(result)) + if result.call_id is not None: + output["tool_call_id"] = result.call_id + if result.name is not None: + output["name"] = result.name + return output + + output["content"] = parts_to_openai_content(message.parts) + return output + + +# --------------------------------------------------------------------------- +# DSPy request -> OpenAI Responses +# +# Responses expects a flat `input` list, not a chat `messages` list. A single +# DSPy assistant message can therefore become two kinds of input items: one +# message item for content and one function_call item for each tool call. +# --------------------------------------------------------------------------- + + +def to_openai_responses_request(request: LMRequest) -> dict[str, Any]: + """Convert a normalized DSPy request into Responses API kwargs.""" + config = request.config + data: dict[str, Any] = { + "model": request.model, + "input": [item for message in request.messages for item in message_to_responses_input_items(message)], + } + data.update(responses_config_kwargs(config, model=request.model)) + if config.tool_choice is not None: + data.update(tool_choice_to_openai(config.tool_choice)) + if request.tools: + data["tools"] = [tool_to_openai(tool) for tool in request.tools] + return data + + +def message_to_responses_input_items(message: LMMessage) -> list[dict[str, Any]]: + """Convert one DSPy message into one or more Responses input items.""" + if message.role == "tool" and len(message.parts) == 1 and isinstance(message.parts[0], LMToolResultPart): + result = message.parts[0] + item = {"type": "function_call_output", "output": responses_tool_output_text(tool_result_to_openai(result)["content"])} + if result.call_id is not None: + item["call_id"] = result.call_id + return [item] + + tool_calls = [part for part in message.parts if isinstance(part, LMToolCallPart)] + content_parts = [part for part in message.parts if not isinstance(part, LMToolCallPart)] + content = parts_to_responses_content(content_parts) + items: list[dict[str, Any]] = [] + + if content or message.role != "assistant" or not tool_calls: + item: dict[str, Any] = {"role": message.role, "content": content} + if message.name is not None: + item["name"] = message.name + items.append(item) + + if message.role == "assistant": + items.extend(tool_call_to_responses_input(tool_call) for tool_call in tool_calls) + return items + + +def parts_to_responses_content(parts: list[Any]) -> list[dict[str, Any]]: + blocks = parts_to_openai_content(parts) + if isinstance(blocks, str): + return [{"type": "input_text", "text": blocks}] + return [content_block_to_responses(block) for block in blocks] + + +def tool_call_to_responses_input(tool_call_part: LMToolCallPart) -> dict[str, Any]: + tool_call = assistant_tool_call_to_openai(tool_call_part) + function = tool_call.get("function", {}) + item = {"type": "function_call", "name": function.get("name", ""), "arguments": function.get("arguments", "{}")} + call_id = tool_call.get("id") or tool_call.get("call_id") + if call_id is not None: + item["call_id"] = call_id + return item + + +def content_block_to_responses(block: dict[str, Any]) -> dict[str, Any]: + block_type = block.get("type") + if block_type == "text": + return {"type": "input_text", "text": block.get("text", "")} + if block_type == "image_url": + image_url = block.get("image_url", {}) + out = {"type": "input_image", "image_url": image_url.get("url", "")} + if image_url.get("detail") is not None: + out["detail"] = image_url["detail"] + return out + if block_type == "input_audio": + return block + if block_type == "file": + file = block.get("file", {}) + return { + "type": "input_file", + "file_data": file.get("file_data"), + "filename": file.get("filename"), + "file_id": file.get("file_id"), + } + return block + + +# --------------------------------------------------------------------------- +# DSPy request -> OpenAI text completions +# +# Text completions have no native message roles. We concatenate text-only +# messages with blank lines and append DSPy's historical response marker. +# --------------------------------------------------------------------------- + + +def to_openai_text_request(request: LMRequest) -> dict[str, Any]: + """Convert a normalized DSPy request into text-completion kwargs.""" + data = {"model": request.model, "prompt": messages_to_text_prompt(request.messages)} + data.update(text_config_kwargs(request.config)) + return data + + +def messages_to_text_prompt(messages: list[LMMessage]) -> str: + """Flatten text-only messages into the prompt used by text completions.""" + chunks = [] + for message in messages: + texts = [] + for part in message.parts: + if not isinstance(part, LMTextPart): + raise ValueError(f"OpenAI text completions only support text parts, but received {type(part).__name__}.") + texts.append(part.text) + chunks.append("".join(texts)) + return "\n\n".join(chunks + ["BEGIN RESPONSE:"]) + + +# --------------------------------------------------------------------------- +# Shared request mappers +# +# The three OpenAI endpoint families use slightly different outer envelopes, but most +# leaf values are the same: text parts, media parts, tools, tool choices, and +# generation config. The helpers below keep those conversions in one place. +# --------------------------------------------------------------------------- + + +def parts_to_openai_content(parts: list[Any]) -> str | list[dict[str, Any]]: + """Convert DSPy parts into OpenAI `content`. + + OpenAI accepts a bare string for the common single-text case. Mixed content + becomes a list of content blocks. + """ + if len(parts) == 1 and isinstance(parts[0], LMTextPart) and "legacy_content_block" not in parts[0].metadata: + return parts[0].text + blocks: list[dict[str, Any]] = [] + for part in parts: + blocks.extend(part_to_openai_blocks(part)) + return blocks + + +def part_to_openai_blocks(part: Any) -> list[dict[str, Any]]: + """Convert one DSPy part into one or more OpenAI content blocks.""" + legacy_block = getattr(part, "metadata", {}).get("legacy_content_block") + if legacy_block is not None: + return [dict(legacy_block)] + if isinstance(part, LMTextPart): + return [{"type": "text", "text": part.text}] + if isinstance(part, LMImagePart): + return [image_to_openai(part)] + if isinstance(part, LMDocumentPart): + return document_to_openai_blocks(part) + if isinstance(part, LMAudioPart): + return [audio_to_openai(part)] + if isinstance(part, LMVideoPart): + return [video_to_openai(part)] + if isinstance(part, LMBinaryPart): + return [binary_to_openai(part)] + if isinstance(part, LMThinkingPart): + return [{"type": "text", "text": part.text}] + if isinstance(part, LMCitationPart): + citation = " ".join(value for value in (part.title, part.text, part.url) if value) + return [{"type": "text", "text": citation}] + if isinstance(part, LMToolResultPart): + return part_to_openai_blocks(LMTextPart(text="".join(part_text(value) for value in part.content))) + return [{"type": "text", "text": str(part)}] + + +def image_to_openai(image: LMImagePart) -> dict[str, Any]: + image_url: dict[str, Any] = {"url": media_source(image)} + if image.detail is not None: + image_url["detail"] = image.detail + return {"type": "image_url", "image_url": image_url} + + +def audio_to_openai(audio: LMAudioPart) -> dict[str, Any]: + if audio.data is not None: + data = audio.data + media_type = audio.media_type + elif audio.path is not None: + data = read_path_base64(audio.path) + media_type = media_type_for_path(audio.path, fallback=audio.media_type) + else: + raise ValueError("OpenAI-format audio input requires base64 `data` or local `path`.") + return {"type": "input_audio", "input_audio": {"data": data, "format": media_format(media_type)}} + + +def document_to_openai_blocks(document: LMDocumentPart) -> list[dict[str, Any]]: + block: dict[str, Any] = {"type": "document"} + if document.source is not None: + block["source"] = document.source + else: + block["source"] = media_source(document) + block["media_type"] = document.media_type + if document.citations: + block["citations"] = document.citations + if document.title is not None: + block["title"] = document.title + if document.context is not None: + block["context"] = document.context + return [block] + + +def video_to_openai(video: LMVideoPart) -> dict[str, Any]: + filename = os.path.basename(video.path) if video.path is not None else None + return binary_to_openai( + LMBinaryPart( + data=video.data, + url=video.url, + file_id=video.file_id, + path=video.path, + media_type=video.media_type, + filename=filename, + ) + ) + + +def binary_to_openai(binary: LMBinaryPart) -> dict[str, Any]: + file_data: dict[str, Any] = {} + if binary.data is not None: + file_data["file_data"] = data_uri(binary.media_type, binary.data) + elif binary.path is not None: + file_data["file_data"] = data_uri_from_path(binary.path, fallback_media_type=binary.media_type) + file_data["filename"] = binary.filename or os.path.basename(binary.path) + elif binary.url is not None: + file_data["file_data"] = binary.url + if binary.file_id is not None: + file_data["file_id"] = binary.file_id + if binary.filename is not None: + file_data["filename"] = binary.filename + return {"type": "file", "file": file_data} + + +def tool_to_openai(tool: LMToolSpec) -> dict[str, Any]: + data = {"type": "function", "function": {"name": tool.name, "parameters": tool.parameters}} + if tool.description is not None: + data["function"]["description"] = tool.description + data.update(tool.provider_data) + return data + + +def tool_choice_to_openai(choice: LMToolChoice) -> dict[str, Any]: + if choice.allowed: + if len(choice.allowed) != 1 or choice.mode not in {"required", "auto"}: + raise ValueError( + "OpenAI-format tool_choice only supports constraining to a single allowed tool " + "with mode 'required' or 'auto'." + ) + data: dict[str, Any] = {"tool_choice": {"type": "function", "function": {"name": choice.allowed[0]}}} + else: + data = {"tool_choice": choice.mode} + if choice.parallel is not None: + data["parallel_tool_calls"] = choice.parallel + return data + + +def assistant_tool_call_to_openai(call: LMToolCallPart) -> dict[str, Any]: + data = {"type": "function", "function": {"name": call.name, "arguments": json.dumps(call.args)}} + if call.id is not None: + data["id"] = call.id + data.update(call.provider_data) + return data + + +def tool_result_to_openai(result: LMToolResultPart) -> dict[str, Any]: + return {"content": parts_to_openai_content(result.content)} + + +def common_config_kwargs(config: LMConfig, *, model: str | None = None, endpoint: str = "chat") -> dict[str, Any]: + """Convert shared DSPy config fields into Chat Completions kwargs.""" + data = dict(config.extensions) + _validate_openai_reasoning_temperature(config, model=model, endpoint=endpoint) + for key in ("temperature", "top_p"): + value = getattr(config, key) + if value is not None: + data[key] = value + if config.max_tokens is not None: + token_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens" + data[token_key] = config.max_tokens + if config.stop: + data["stop"] = config.stop + if config.logprobs is not None: + data["logprobs"] = config.logprobs + if config.n is not None: + data["n"] = config.n + if config.response_format is not None: + data["response_format"] = config.response_format + if config.reasoning is not None: + data.update(reasoning_to_chat_kwargs(config.reasoning)) + if config.prompt_cache is not None: + data.update(prompt_cache_to_kwargs(config.prompt_cache)) + return data + + +def responses_config_kwargs(config: LMConfig, *, model: str | None = None) -> dict[str, Any]: + """Convert shared DSPy config fields into Responses API kwargs.""" + data = dict(config.extensions) if config.extensions else {} + _validate_openai_reasoning_temperature(config, model=model, endpoint="responses") + for key in ("temperature", "top_p"): + value = getattr(config, key) + if value is not None: + data[key] = value + if config.max_tokens is not None: + data["max_output_tokens"] = config.max_tokens + if config.n is not None: + data["n"] = config.n + if config.logprobs is not None: + data["logprobs"] = config.logprobs + if config.stop: + data["stop"] = config.stop + if config.reasoning is not None: + data.update(reasoning_to_responses_kwargs(config.reasoning)) + if config.prompt_cache is not None: + data.update(prompt_cache_to_kwargs(config.prompt_cache)) + if config.response_format is not None: + text = data.pop("text", {}) + data["text"] = {**text, "format": response_format_to_responses(config.response_format)} + return data + + +def text_config_kwargs(config: LMConfig) -> dict[str, Any]: + """Convert shared DSPy config fields into text-completion kwargs.""" + data = dict(config.extensions) + for key in ("temperature", "max_tokens", "top_p"): + value = getattr(config, key) + if value is not None: + data[key] = value + if config.stop: + data["stop"] = config.stop + if config.logprobs is not None: + data["logprobs"] = config.logprobs + if config.n is not None: + data["n"] = config.n + return data + + +def reasoning_to_chat_kwargs(reasoning: Any) -> dict[str, Any]: + data = {} + if reasoning.effort is not None: + data["reasoning_effort"] = reasoning.effort + return data + + +def reasoning_to_responses_kwargs(reasoning: Any) -> dict[str, Any]: + data = {} + if reasoning.effort is not None: + data["effort"] = reasoning.effort + if reasoning.summary is not None: + data["summary"] = reasoning.summary + return {"reasoning": data} if data else {} + + +def _validate_openai_reasoning_temperature(config: LMConfig, *, model: str | None, endpoint: str) -> None: + if not _is_openai_reasoning_model(model): + return + effort = getattr(config.reasoning, "effort", None) if config.reasoning is not None else None + if effort in {None, "none"}: + return + if config.temperature in {None, 1}: + return + + from dspy.utils.exceptions import LMUnsupportedFeatureError + + raise LMUnsupportedFeatureError( + "OpenAI reasoning models only support the default temperature when reasoning effort is active. " + "Use temperature=None or temperature=1, or set reasoning_effort='none'.", + model=model, + provider="openai", + features=["temperature", "reasoning"], + issues=[ + f"{endpoint} request used reasoning effort {effort!r} with temperature={config.temperature!r}.", + ], + ) + + +def _uses_max_completion_tokens(model: str | None) -> bool: + return _is_openai_reasoning_model(model) + + +def _is_openai_reasoning_model(model: str | None) -> bool: + if not isinstance(model, str): + return False + model_name = model.removeprefix("openai/").lower() + if "chat" in model_name: + return False + return model_name.startswith(("o1", "o3", "o4", "gpt-5")) + + +def prompt_cache_to_kwargs(cache: Any) -> dict[str, Any]: + data = {} + if cache.key is not None: + data["prompt_cache_key"] = cache.key + if cache.enabled is False: + data["prompt_cache"] = False + return data + + +def response_format_to_responses(value: Any) -> Any: + if isinstance(value, type) and issubclass(value, pydantic.BaseModel): + return {"name": value.__name__, "type": "json_schema", "schema": value.model_json_schema()} + return value + + +def responses_tool_output_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join( + block.get("text", "") if isinstance(block, dict) and block.get("type") in {"text", "input_text"} else str(block) + for block in content + ) + return str(content) + + +# --------------------------------------------------------------------------- +# OpenAI response -> DSPy response +# +# Chat and text completions return `choices`; Responses returns one `output` +# list. Both are normalized to an `LMResponse` with one or more `LMOutput`s and +# typed DSPy parts. +# --------------------------------------------------------------------------- + + +def completion_to_lm_response(response: Any, request: LMRequest) -> LMResponse: + """Convert an OpenAI Chat or text completion response into `LMResponse`.""" + choices = get_value(response, "choices", []) or [] + return LMResponse( + model=get_value(response, "model") or request.model, + outputs=[choice_to_lm_output(choice) for choice in choices], + usage=usage_from_response(response), + cache_hit=bool(get_value(response, "cache_hit", False)), + response_id=get_value(response, "id"), + provider_response=response, + ) + + +def choice_to_lm_output(choice: Any) -> LMOutput: + """Convert one completion choice into one DSPy output candidate.""" + message = get_value(choice, "message") + parts = [] + if message is not None: + reasoning = get_value(message, "reasoning_content") + if reasoning: + parts.append(LMThinkingPart(text=str(reasoning))) + content = get_value(message, "content") + if content: + parts.extend(message_content_to_parts(content)) + for tool_call in get_value(message, "tool_calls") or []: + parts.append(provider_tool_call_to_part(tool_call)) + parts.extend(extract_citations_from_choice(choice)) + else: + text = get_value(choice, "text") + if text: + parts.extend(message_content_to_parts(text)) + finish_reason = get_value(choice, "finish_reason") + return LMOutput( + parts=parts, + finish_reason=finish_reason, + truncated=finish_reason == "length", + logprobs=get_value(choice, "logprobs"), + provider_output=choice, + ) + + +def responses_to_lm_response(response: Any, request: LMRequest) -> LMResponse: + """Convert an OpenAI Responses object into `LMResponse`. + + The Responses API represents one assistant answer as a sequence of output + items: messages, function calls, reasoning, binary artifacts, images, and refusals. + DSPy stores those as typed parts on one `LMOutput`. + """ + parts = [] + for output_item in get_value(response, "output", []) or []: + output_type = get_value(output_item, "type") + if output_type == "message": + for content_item in get_value(output_item, "content", []) or []: + parts.extend(response_content_item_to_parts(content_item)) + parts.extend(responses_annotations_to_citations(content_item)) + elif output_type == "function_call": + parts.append(responses_function_call_to_part(output_item)) + elif output_type in {"image", "output_image", "image_generation_call"}: + parts.append(output_image_to_part(output_item)) + elif output_type in {"audio", "output_audio"}: + parts.append(output_audio_to_part(output_item)) + elif output_type in {"file", "output_file"}: + parts.append(output_file_to_part(output_item)) + elif output_type == "refusal": + parts.append(refusal_to_part(output_item)) + elif output_type == "reasoning": + for item in get_value(output_item, "content") or get_value(output_item, "summary") or []: + text = get_value(item, "text") + if text: + parts.append(LMThinkingPart(text=text)) + return LMResponse( + model=get_value(response, "model") or request.model, + outputs=[LMOutput(parts=parts, provider_output=response)], + usage=usage_from_response(response), + cache_hit=bool(get_value(response, "cache_hit", False)), + response_id=get_value(response, "id"), + provider_response=response, + ) + + +def message_content_to_parts(content: Any) -> list[Any]: + if isinstance(content, str): + return [LMTextPart(text=content)] + if not isinstance(content, list): + return [LMTextPart(text=str(content))] + parts = [] + for item in content: + parts.extend(response_content_item_to_parts(item)) + return parts + + +def response_content_item_to_parts(item: Any) -> list[Any]: + item_type = get_value(item, "type") + text = get_value(item, "text") + if item_type in {"text", "output_text", "input_text"} or (text is not None and item_type is None): + return [LMTextPart(text=text)] + if item_type in {"refusal", "output_refusal"}: + return [refusal_to_part(item)] + if item_type in {"image", "output_image", "image_url"}: + return [output_image_to_part(item)] + if item_type in {"audio", "output_audio", "input_audio"}: + return [output_audio_to_part(item)] + if item_type in {"file", "output_file", "input_file"}: + return [output_file_to_part(item)] + if item_type in {"tool_call", "function_call"}: + return [provider_tool_call_to_part(item)] + return [] + + +def provider_tool_call_to_part(tool_call: Any) -> LMToolCallPart: + """Convert an OpenAI-shaped tool call into a DSPy tool-call part.""" + function = get_value(tool_call, "function", {}) + name = get_value(function, "name") or get_value(tool_call, "name") + arguments = get_value(function, "arguments", get_value(tool_call, "arguments", "{}")) + provider_data = model_dump(tool_call) + try: + args = json.loads(arguments) if isinstance(arguments, str) else dict(arguments) + except Exception as error: + args = {} + provider_data["raw_arguments"] = arguments + provider_data["arguments_parse_error"] = str(error) + call_id = get_value(tool_call, "call_id") or get_value(tool_call, "id") + return LMToolCallPart(id=call_id, name=name or "", args=args, provider_data=provider_data) + + +def responses_function_call_to_part(output_item: Any) -> LMToolCallPart: + """Convert one Responses function_call item into a DSPy tool-call part.""" + args = get_value(output_item, "arguments", {}) + provider_data = model_dump(output_item) + if isinstance(args, str): + try: + args = json.loads(args) + except Exception as error: + provider_data["raw_arguments"] = args + provider_data["arguments_parse_error"] = str(error) + args = {} + return LMToolCallPart(id=get_value(output_item, "call_id"), name=get_value(output_item, "name", ""), args=args, provider_data=provider_data) + + +def citation_to_part(citation: Any) -> LMCitationPart: + if hasattr(citation, "model_dump"): + citation = citation.model_dump(exclude_none=True) + if not isinstance(citation, dict): + citation = {"text": str(citation)} + citation_fields = {"cited_text", "text", "supported_text", "document_title", "title", "url"} + return LMCitationPart( + text=citation.get("cited_text") or citation.get("text") or citation.get("supported_text"), + title=citation.get("document_title") or citation.get("title"), + url=citation.get("url"), + metadata={key: value for key, value in citation.items() if key not in citation_fields}, + ) + + +def extract_citations_from_choice(choice: Any) -> list[LMCitationPart]: + try: + message = get_value(choice, "message") + provider_specific_fields = get_value(message, "provider_specific_fields", {}) or {} + citations_data = provider_specific_fields.get("citations") + if isinstance(citations_data, list): + citations = [] + for item in citations_data: + citations.extend(item if isinstance(item, list) else [item]) + return [citation_to_part(citation) for citation in citations] + except Exception: + return [] + return [] + + +def responses_annotations_to_citations(content_item: Any) -> list[LMCitationPart]: + return [citation_to_part(annotation) for annotation in get_value(content_item, "annotations", []) or []] + + +def output_image_to_part(value: Any) -> LMImagePart: + data = model_dump(value) + image_url = data.get("image_url") + if isinstance(image_url, dict): + image_url = image_url.get("url") + source = image_url or data.get("url") + b64_data = data.get("b64_json") or data.get("data") + file_id = data.get("file_id") + media_type = data.get("media_type") or data.get("mime_type") or "image/png" + detail = data.get("detail") + if b64_data is not None: + if isinstance(b64_data, str) and b64_data.startswith("data:"): + media_type, b64_data = split_data_uri(b64_data) + return LMImagePart(data=b64_data, media_type=media_type, detail=detail) + if source is not None: + return LMImagePart(url=source, media_type=media_type, detail=detail) + if file_id is not None: + return LMImagePart(file_id=file_id, media_type=media_type, detail=detail) + raise ValueError("Provider image output did not include data, url, or file_id.") + + +def output_audio_to_part(value: Any) -> LMAudioPart: + data = model_dump(value) + audio = data.get("audio") if isinstance(data.get("audio"), dict) else data + source = audio.get("url") + b64_data = audio.get("data") or audio.get("b64_json") + file_id = audio.get("file_id") + media_type = audio.get("media_type") or audio.get("mime_type") or "audio/wav" + if b64_data is not None: + if isinstance(b64_data, str) and b64_data.startswith("data:"): + media_type, b64_data = split_data_uri(b64_data) + return LMAudioPart(data=b64_data, media_type=media_type) + if source is not None: + return LMAudioPart(url=source, media_type=media_type) + if file_id is not None: + return LMAudioPart(file_id=file_id, media_type=media_type) + raise ValueError("Provider audio output did not include data, url, or file_id.") + + +def output_file_to_part(value: Any) -> LMBinaryPart: + data = model_dump(value) + file = data.get("file") if isinstance(data.get("file"), dict) else data + source = file.get("url") + b64_data = file.get("file_data") or file.get("data") + file_id = file.get("file_id") or file.get("id") + filename = file.get("filename") + media_type = file.get("media_type") or file.get("mime_type") or "application/octet-stream" + if b64_data is not None: + if isinstance(b64_data, str) and b64_data.startswith("data:"): + media_type, b64_data = split_data_uri(b64_data) + return LMBinaryPart(data=b64_data, media_type=media_type, filename=filename) + if source is not None: + return LMBinaryPart(url=source, media_type=media_type, filename=filename) + if file_id is not None: + return LMBinaryPart(file_id=file_id, media_type=media_type, filename=filename) + raise ValueError("Provider file output did not include data, url, or file_id.") + + +def refusal_to_part(value: Any) -> LMRefusalPart: + text = get_value(value, "refusal") or get_value(value, "text") or get_value(value, "content") or str(value) + return LMRefusalPart(text=str(text)) + + +def cost_from_response(response: Any) -> float | None: + hidden = getattr(response, "_hidden_params", None) or {} + return hidden.get("response_cost") if isinstance(hidden, dict) else None + + +def usage_from_response(response: Any) -> LMUsage | None: + """Convert provider usage objects or dictionaries into `LMUsage`.""" + usage = get_value(response, "usage") + if usage is None: + return None + if hasattr(usage, "model_dump"): + usage = usage.model_dump(exclude_none=True) + elif not isinstance(usage, dict): + data = {} + for key in dir(usage): + if key.startswith("_"): + continue + try: + value = getattr(usage, key) + except Exception: + continue + if value is not None and not callable(value): + data[key] = value + usage = data + return LMUsage(**dict(usage)) + + +# --------------------------------------------------------------------------- +# Small utilities +# --------------------------------------------------------------------------- + + +def media_source(part: LMImagePart | LMAudioPart | LMDocumentPart | LMBinaryPart) -> str: + if part.data is not None: + return data_uri(part.media_type, part.data) + if part.url is not None: + return part.url + if part.file_id is not None: + return part.file_id + if part.path is not None: + return data_uri_from_path(part.path, fallback_media_type=part.media_type) + raise ValueError(f"{type(part).__name__} has no media source.") + + +def read_path_base64(path: str) -> str: + with open(path, "rb") as file: + return base64.b64encode(file.read()).decode("ascii") + + +def media_type_for_path(path: str, *, fallback: str) -> str: + return mimetypes.guess_type(path)[0] or fallback + + +def data_uri_from_path(path: str, *, fallback_media_type: str) -> str: + return data_uri(media_type_for_path(path, fallback=fallback_media_type), read_path_base64(path)) + + +def data_uri(media_type: str, data: str) -> str: + if data.startswith("data:"): + return data + return f"data:{media_type};base64,{data}" + + +def split_data_uri(value: str) -> tuple[str, str]: + if not value.startswith("data:") or "," not in value: + return "application/octet-stream", value + header, data = value.split(",", 1) + media_type = header.removeprefix("data:").split(";", 1)[0] + return media_type, data + + +def media_format(media_type: str) -> str: + format_ = media_type.split("/", 1)[1] if "/" in media_type else media_type + return {"x-wav": "wav", "mpeg": "mp3"}.get(format_, format_) + + +def part_text(value: Any) -> str: + return value.text if isinstance(value, LMTextPart) else str(value) + + +def get_value(value: Any, key: str, default: Any = None) -> Any: + return value.get(key, default) if isinstance(value, dict) else getattr(value, key, default) + + +def model_dump(value: Any) -> dict[str, Any]: + if hasattr(value, "model_dump"): + return value.model_dump(exclude_none=True) + if isinstance(value, dict): + return dict(value) + data = {} + for key in ("id", "call_id", "type", "name", "arguments", "status", "text", "refusal", "url", "data", "file_id", "filename", "media_type", "mime_type"): + item = getattr(value, key, None) + if item is not None: + data[key] = item + return data + + +# --------------------------------------------------------------------------- +# Current BaseLM adapter compatibility helpers +# +# These helpers keep today's adapter->BaseLM boundary normalized internally while +# BaseLM still returns legacy `list[str | dict | None]` outputs. Future native +# LanguageModel implementations should use completion_to_lm_response() / +# responses_to_lm_response() directly. +# --------------------------------------------------------------------------- + + +def lm_response_from_legacy_outputs(outputs: list[dict[str, Any] | str | None], request: LMRequest) -> LMResponse: + """Normalize current legacy `BaseLM` outputs into an `LMResponse`.""" + if not outputs: + return LMResponse(model=request.model, outputs=[LMOutput(parts=[], metadata={"empty_legacy_outputs": True})]) + return LMResponse(model=request.model, outputs=[lm_output_from_legacy_output(output) for output in outputs]) + + +def lm_output_from_legacy_output(output: dict[str, Any] | str | None) -> LMOutput: + """Normalize one current legacy `BaseLM` output item into an `LMOutput`.""" + if isinstance(output, str): + return LMOutput(parts=[LMTextPart(text=output)], provider_output=output) + if output is None: + return LMOutput(parts=[]) + + parts = [] + text = output.get("text") + if text: + parts.append(LMTextPart(text=text)) + reasoning = output.get("reasoning_content") + if reasoning: + parts.append(LMThinkingPart(text=str(reasoning))) + for tool_call in output.get("tool_calls") or []: + parts.append(provider_tool_call_to_part(tool_call)) + for citation in output.get("citations") or []: + parts.append(citation_to_part(citation)) + return LMOutput(parts=parts, logprobs=output.get("logprobs"), provider_output=output) diff --git a/dspy/core/types.py b/dspy/core/types.py new file mode 100644 index 0000000000..6ceb9eb447 --- /dev/null +++ b/dspy/core/types.py @@ -0,0 +1,2000 @@ +"""Normalized request, response, and stream types for DSPy language models.""" + +from __future__ import annotations + +import json +import mimetypes +from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from dataclasses import dataclass +from dataclasses import field as dataclass_field +from pprint import pformat +from typing import Annotated, Any, Literal +from urllib.parse import urlparse + +import pydantic +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class LMBasePart(BaseModel): + """A single content item in an LM message or output.""" + + type: str + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class LMSourcePart(LMBasePart): + """Content from exactly one provider-addressable source.""" + + media_type: str + data: str | None = None + url: str | None = None + file_id: str | None = None + path: str | None = None + + @model_validator(mode="after") + def validate_one_source(self) -> LMSourcePart: + _validate_one_source(self) + return self + + +class LMTextPart(LMBasePart): + """Text content.""" + + type: Literal["text"] = "text" + text: str + + +class LMImagePart(LMSourcePart): + """Image content from data, a URL, a file ID, or a local path.""" + + type: Literal["image"] = "image" + media_type: str = "image/png" + detail: Literal["low", "high", "auto"] | None = None + + +class LMAudioPart(LMSourcePart): + """Audio content from data, a URL, a file ID, or a local path.""" + + type: Literal["audio"] = "audio" + media_type: str = "audio/wav" + + +class LMVideoPart(LMSourcePart): + """Video content from data, a URL, a file ID, or a local path.""" + + type: Literal["video"] = "video" + media_type: str = "video/mp4" + + +class LMDocumentPart(LMBasePart): + """Semantic source/document content, optionally citation-enabled. + + Documents are source material: text, PDFs, reports, contracts, or other + provider-addressable evidence that benefits from title/context/citation + semantics. Use `LMBinaryPart` for opaque attachments or arbitrary bytes. + """ + + type: Literal["document"] = "document" + media_type: str = "application/pdf" + data: str | None = None + url: str | None = None + file_id: str | None = None + path: str | None = None + source: dict[str, Any] | None = None + citations: dict[str, Any] = Field(default_factory=dict) + title: str | None = None + context: str | None = None + + @model_validator(mode="after") + def validate_source(self) -> LMDocumentPart: + has_media_source = any(value is not None for value in (self.data, self.url, self.file_id, self.path)) + if self.source is not None and has_media_source: + raise ValueError("LMDocumentPart accepts either source or one of data, url, file_id, or path, not both.") + if self.source is None: + _validate_one_source(self) + elif not self.source: + raise ValueError("LMDocumentPart.source must be non-empty when provided.") + return self + + +class LMBinaryPart(LMSourcePart): + """Opaque binary content from data, a URL, a file ID, or a local path.""" + + type: Literal["binary"] = "binary" + media_type: str = "application/octet-stream" + filename: str | None = None + + +class LMToolCallPart(LMBasePart): + """A model request to call a tool. + + Use `dspy.ToolCall(...)` as a shorter public alias when constructing + assistant messages by hand. + + Args: + id: Provider call ID, when the backend uses one. + name: Name of the tool to call. + args: JSON-like arguments for the tool. + provider_data: Raw provider fields to keep with the tool call. + + Examples: + ```python + import dspy + + assistant = dspy.Assistant( + dspy.ToolCall( + id="call_1", + name="search", + args={"query": "DSPy"}, + ) + ) + ``` + """ + + type: Literal["tool_call"] = "tool_call" + id: str | None = None + name: str + args: dict[str, Any] = Field(default_factory=dict) + provider_data: dict[str, Any] = Field(default_factory=dict) + + +class LMToolResultPart(LMBasePart): + """A tool execution result sent back to a model.""" + + type: Literal["tool_result"] = "tool_result" + call_id: str | None = None + name: str | None = None + content: list[LMPart] = Field(default_factory=list) + is_error: bool = False + provider_data: dict[str, Any] = Field(default_factory=dict) + + @model_validator(mode="before") + @classmethod + def normalize_content(cls, data: Any) -> Any: + if isinstance(data, dict) and "content" in data: + data = dict(data) + content = data["content"] + if content is None: + data["content"] = [] + elif isinstance(content, list): + data["content"] = [_coerce_part(item) for item in content] + else: + data["content"] = [_coerce_part(content)] + return data + + +class LMThinkingPart(LMBasePart): + """Reasoning or thinking content returned by a model.""" + + type: Literal["thinking"] = "thinking" + text: str + redacted: bool = False + + +class LMCitationPart(LMBasePart): + """A source citation returned by a model.""" + + type: Literal["citation"] = "citation" + text: str | None = None + title: str | None = None + url: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_has_content(self) -> LMCitationPart: + if self.text is None and self.title is None and self.url is None: + raise ValueError("LMCitationPart requires at least one of text, title, or url.") + return self + + +class LMRefusalPart(LMBasePart): + """A model refusal.""" + + type: Literal["refusal"] = "refusal" + text: str + + +LMPart = Annotated[ + LMTextPart + | LMImagePart + | LMAudioPart + | LMVideoPart + | LMDocumentPart + | LMBinaryPart + | LMToolCallPart + | LMToolResultPart + | LMThinkingPart + | LMCitationPart + | LMRefusalPart, + Field(discriminator="type"), +] + + +class LMMessage(BaseModel): + """A role-attributed sequence of LM parts.""" + + role: str + parts: list[LMPart] + name: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="before") + @classmethod + def normalize_parts(cls, data: Any) -> Any: + if isinstance(data, cls): + return data + if isinstance(data, dict): + data = dict(data) + if data.get("role") == "tool" and "parts" not in data: + content = data.pop("content", None) + call_id = data.pop("tool_call_id", None) + name = data.pop("name", None) + data["parts"] = [ + LMToolResultPart( + call_id=call_id, + name=name, + content=_parts_from_openai_content(content), + ) + ] + elif "parts" not in data: + parts = _parts_from_openai_content(data.pop("content", None)) if "content" in data else [] + if "tool_calls" in data: + parts.extend(_tool_calls_from_openai(data.pop("tool_calls") or [])) + data["parts"] = parts + else: + data["parts"] = [_coerce_part(part) for part in data["parts"]] + if "tool_calls" in data: + data["parts"].extend(_tool_calls_from_openai(data.pop("tool_calls") or [])) + return data + + @property + def text(self) -> str | None: + texts = [part.text for part in self.parts if isinstance(part, LMTextPart)] + return "".join(texts) if texts else None + + +class LMToolSpec(BaseModel): + """Provider-independent schema for a tool available to an LM.""" + + type: Literal["function"] = "function" + name: str + description: str | None = None + parameters: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + provider_data: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class LMReasoningConfig(BaseModel): + """Reasoning controls for models with native reasoning support.""" + + effort: str | None = None + max_tokens: int | None = None + summary: str | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_value(cls, value: Any = None, **overrides: Any) -> LMReasoningConfig: + data = _config_data(value, str_field="effort") + if "reasoning_effort" in data and "effort" not in data: + data["effort"] = data.pop("reasoning_effort") + data.update({key: value for key, value in overrides.items() if value is not _MISSING}) + return cls(**data) + + +class LMToolChoice(BaseModel): + """Tool-choice controls for native tool-capable models.""" + + mode: Literal["auto", "required", "none"] = "auto" + # Tool names that are allowed for a model request. + allowed: list[str] | None = None + parallel: bool | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_value(cls, value: Any = None, **overrides: Any) -> LMToolChoice: + if isinstance(value, Mapping) and value.get("type") == "function" and isinstance(value.get("function"), Mapping): + # Accept OpenAI/LiteLLM-shaped tool choices for the current BaseLM + # compatibility path. Internally, DSPy represents this as requiring + # one allowed tool. + data = {"mode": "required", "allowed": [value["function"].get("name", "")]} + else: + data = _config_data(value, str_field="mode") + if "parallel_tool_calls" in data and "parallel" not in data: + data["parallel"] = data.pop("parallel_tool_calls") + data.update({key: value for key, value in overrides.items() if value is not _MISSING}) + return cls(**data) + + +class LMCacheConfig(BaseModel): + """DSPy memoization cache controls for a normalized LM request. + + This cache skips the provider call entirely when DSPy finds an exact + request match. Use `LMPromptCacheConfig` for provider-side prompt/token + caching that still sends the request to the provider. + """ + + enabled: bool | None = None + rollout_id: int | str | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_value(cls, value: Any = None, **overrides: Any) -> LMCacheConfig: + data = _config_data(value, bool_field="enabled") + data.update({key: value for key, value in overrides.items() if value is not _MISSING}) + return cls(**data) + + +class LMPromptCacheConfig(BaseModel): + """Provider-side prompt/token cache controls. + + Prompt caching is not DSPy memoization. The provider call still happens, + but the backend may reuse cached prompt prefixes or KV state for lower + latency or lower input-token cost. + """ + + enabled: bool | None = None + key: str | None = None + + model_config = ConfigDict(extra="forbid") + + @classmethod + def from_value(cls, value: Any = None, **overrides: Any) -> LMPromptCacheConfig: + data = _config_data(value, bool_field="enabled") + if "prompt_cache_key" in data and "key" not in data: + data["key"] = data.pop("prompt_cache_key") + data.update({key: value for key, value in overrides.items() if value is not _MISSING}) + return cls(**data) + + +_MISSING = object() + + +def _config_data(value: Any, *, str_field: str | None = None, bool_field: str | None = None) -> dict[str, Any]: + if value is None: + return {} + if isinstance(value, BaseModel): + return value.model_dump(exclude_none=True) + if isinstance(value, Mapping): + return dict(value) + if str_field is not None and isinstance(value, str): + return {str_field: value} + if bool_field is not None and isinstance(value, bool): + return {bool_field: value} + raise TypeError(f"Cannot convert {type(value)!r} to a config object.") + + +# Keep existing `dspy.LM(...)` / `lm(...)` keyword aliases in the call compatibility path. +_KNOWN_CONFIG_KEYS = { + "temperature", + "max_tokens", + "top_p", + "stop", + "n", + "logprobs", + "response_format", + "reasoning", + "reasoning_effort", + "tool_choice", + "parallel_tool_calls", + "cache", + "rollout_id", + "prompt_cache", + "prompt_cache_key", + "extensions", +} + + +def _split_config_kwargs(kwargs: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + data: dict[str, Any] = {} + extensions = dict(kwargs.get("extensions", {}) or {}) + + for key, value in kwargs.items(): + if key == "extensions": + continue + if key in _KNOWN_CONFIG_KEYS: + data[key] = value + else: + extensions[key] = value + return data, extensions + + +def _normalize_reasoning_config(data: dict[str, Any]) -> None: + if "reasoning_effort" in data: + effort = data.pop("reasoning_effort") + if data.get("reasoning") is not None or effort is not None: + data["reasoning"] = LMReasoningConfig.from_value(data.get("reasoning"), effort=effort) + elif "reasoning" in data and data["reasoning"] is not None: + data["reasoning"] = LMReasoningConfig.from_value(data["reasoning"]) + + +def _normalize_tool_choice_config(data: dict[str, Any]) -> None: + parallel = data.pop("parallel_tool_calls", _MISSING) + if data.get("tool_choice") is not None or parallel not in (_MISSING, None): + data["tool_choice"] = LMToolChoice.from_value(data.get("tool_choice"), parallel=parallel) + + +def _normalize_cache_config(data: dict[str, Any]) -> None: + rollout_id = data.pop("rollout_id", _MISSING) + if data.get("cache") is not None or rollout_id not in (_MISSING, None): + data["cache"] = LMCacheConfig.from_value(data.get("cache"), rollout_id=rollout_id) + + +def _normalize_prompt_cache_config(data: dict[str, Any]) -> None: + prompt_cache_key = data.pop("prompt_cache_key", _MISSING) + if data.get("prompt_cache") is not None or prompt_cache_key not in (_MISSING, None): + data["prompt_cache"] = LMPromptCacheConfig.from_value(data.get("prompt_cache"), key=prompt_cache_key) + + +class LMConfig(BaseModel): + """Common generation controls for an LM request.""" + + temperature: float | None = None + max_tokens: int | None = None + top_p: float | None = None + stop: list[str] | None = None + n: int | None = None + logprobs: bool | int | None = None + response_format: Any | None = None + reasoning: LMReasoningConfig | None = None + tool_choice: LMToolChoice | None = None + cache: LMCacheConfig | None = None + prompt_cache: LMPromptCacheConfig | None = None + extensions: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + @classmethod + def from_kwargs(cls, **kwargs: Any) -> LMConfig: + data, extensions = _split_config_kwargs(kwargs) + _normalize_reasoning_config(data) + _normalize_tool_choice_config(data) + _normalize_cache_config(data) + _normalize_prompt_cache_config(data) + data["extensions"] = extensions + return cls(**data) + + +def _merge_lm_config(left: LMConfig | None, right: LMConfig | None) -> LMConfig | None: + if left is None: + return right + if right is None: + return left + + data = left.model_dump() + right_data = right.model_dump(exclude_none=True) + for key in ("reasoning", "tool_choice", "cache", "prompt_cache"): + if key in right_data and isinstance(data.get(key), dict) and isinstance(right_data[key], dict): + right_data[key] = {**data[key], **right_data[key]} + extensions = {**left.extensions, **right.extensions} + data.update(right_data) + data["extensions"] = extensions + return LMConfig(**data) + + +def _merge_config_overrides(config: LMConfig, kwargs: dict[str, Any]) -> LMConfig: + data = config.model_dump() + extensions = dict(config.extensions) + + direct_keys = { + "temperature", + "max_tokens", + "top_p", + "stop", + "n", + "logprobs", + "response_format", + } + for key in direct_keys & kwargs.keys(): + data[key] = kwargs[key] + + if "reasoning" in kwargs or "reasoning_effort" in kwargs: + data["reasoning"] = LMReasoningConfig.from_value( + kwargs.get("reasoning", data.get("reasoning")), + effort=kwargs.get("reasoning_effort", _MISSING), + ) + + if "tool_choice" in kwargs or "parallel_tool_calls" in kwargs: + data["tool_choice"] = LMToolChoice.from_value( + kwargs.get("tool_choice", data.get("tool_choice")), + parallel=kwargs.get("parallel_tool_calls", _MISSING), + ) + + if "cache" in kwargs or "rollout_id" in kwargs: + data["cache"] = LMCacheConfig.from_value( + kwargs.get("cache", data.get("cache")), + rollout_id=kwargs.get("rollout_id", _MISSING), + ) + + if "prompt_cache" in kwargs or "prompt_cache_key" in kwargs: + data["prompt_cache"] = LMPromptCacheConfig.from_value( + kwargs.get("prompt_cache", data.get("prompt_cache")), + key=kwargs.get("prompt_cache_key", _MISSING), + ) + + if "extensions" in kwargs: + extra = kwargs["extensions"] + if extra is None: + extensions = {} + elif isinstance(extra, Mapping): + extensions.update(extra) + else: + raise TypeError("`extensions` override must be a mapping or None.") + + handled = _KNOWN_CONFIG_KEYS + for key, value in kwargs.items(): + if key not in handled: + extensions[key] = value + data["extensions"] = extensions + return LMConfig(**data) + + +@dataclass +class LMRequestPatch: + """A partial normalized LM request contributed while rendering a DSPy call. + + `LMRequest` is the complete object a `LanguageModel` receives. A patch is + the smaller, composable unit that DSPy type strategies can contribute while + an adapter is still building that request: extra messages, extra parts, + native tools, native config, or signature fields that should be hidden from + the outer adapter's ordinary text/JSON/XML rendering. + """ + + messages: list[LMMessage] = dataclass_field(default_factory=list) + system_parts: list[LMPart] = dataclass_field(default_factory=list) + user_parts: list[LMPart] = dataclass_field(default_factory=list) + assistant_parts: list[LMPart] = dataclass_field(default_factory=list) + tools: list[LMToolSpec] = dataclass_field(default_factory=list) + config: LMConfig | None = None + delete_input_fields: tuple[str, ...] = () + delete_output_fields: tuple[str, ...] = () + metadata: dict[str, Any] = dataclass_field(default_factory=dict) + + def merge(self, other: LMRequestPatch) -> LMRequestPatch: + """Return a new patch containing this patch followed by `other`.""" + return LMRequestPatch( + messages=[*self.messages, *other.messages], + system_parts=[*self.system_parts, *other.system_parts], + user_parts=[*self.user_parts, *other.user_parts], + assistant_parts=[*self.assistant_parts, *other.assistant_parts], + tools=[*self.tools, *other.tools], + config=_merge_lm_config(self.config, other.config), + delete_input_fields=(*self.delete_input_fields, *other.delete_input_fields), + delete_output_fields=(*self.delete_output_fields, *other.delete_output_fields), + metadata={**self.metadata, **other.metadata}, + ) + + def as_lm_kwargs(self) -> dict[str, Any]: + """Return the legacy kwargs implied by this patch. + + This keeps the first implementation usable with today's adapter call + path, which still passes `lm_kwargs` rather than an `LMRequestPatch` all + the way down. Message and part patches are intentionally not flattened + here; they require the next adapter-call refactor. + """ + kwargs = self.config.model_dump(exclude_none=True) if self.config is not None else {} + extensions = kwargs.pop("extensions", {}) or {} + kwargs = {**extensions, **kwargs} + if self.tools: + kwargs["tools"] = list(self.tools) + return kwargs + + +class LMRequest(BaseModel): + """A normalized request passed to a `LanguageModel`.""" + + model: str + messages: list[LMMessage] + tools: list[LMToolSpec] = Field(default_factory=list) + config: LMConfig = Field(default_factory=LMConfig) + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + @classmethod + def from_call( + cls, + *, + model: str, + items: tuple[Any, ...] = (), + prompt: str | None = None, + messages: list[dict[str, Any] | LMMessage] | None = None, + tools: list[Any] | None = None, + **kwargs: Any, + ) -> LMRequest: + if messages is not None and (items or prompt is not None): + raise ValueError("Pass messages or direct-call inputs, not both.") + + collected_tools: list[Any] = list(tools or []) + if messages is not None: + normalized_messages = [_coerce_message(message) for message in messages] + else: + normalized_messages, positional_tools = _messages_from_items(items, prompt=prompt) + collected_tools.extend(positional_tools) + + config = LMConfig.from_kwargs(**kwargs) + return cls( + model=model, + messages=normalized_messages, + tools=[_coerce_tool_spec(tool) for tool in collected_tools], + config=config, + ) + + @classmethod + def from_prompt_or_messages( + cls, + *, + model: str, + prompt: str | None = None, + messages: list[dict[str, Any] | LMMessage] | None = None, + **kwargs: Any, + ) -> LMRequest: + return cls.from_call(model=model, prompt=prompt, messages=messages, **kwargs) + + def with_config_overrides(self, **kwargs: Any) -> LMRequest: + """Return a copy with explicit request config overrides applied. + + Only fields implied by the supplied keyword arguments are changed. This + preserves existing grouped config such as `cache`, `prompt_cache`, + `tool_choice`, `reasoning`, `stop`, and provider-specific + `extensions` when an unrelated setting is overridden. + """ + if not kwargs: + return self + merged = _merge_config_overrides(self.config, kwargs) + return self.model_copy(update={"config": merged}, deep=True) + + +class LMUsage(BaseModel): + """Token and timing usage for one LM request. + + Both DSPy token names and provider token names are populated because both + are existing user-visible history/usage interfaces. + """ + + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + prompt_tokens: int | None = None + completion_tokens: int | None = None + reasoning_tokens: int | None = None + cache_read_tokens: int | None = None + cache_write_tokens: int | None = None + input_audio_tokens: int | None = None + output_audio_tokens: int | None = None + details: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="allow") + + @model_validator(mode="after") + def fill_aliases(self) -> LMUsage: + if self.input_tokens is None and self.prompt_tokens is not None: + self.input_tokens = self.prompt_tokens + if self.output_tokens is None and self.completion_tokens is not None: + self.output_tokens = self.completion_tokens + if self.prompt_tokens is None and self.input_tokens is not None: + self.prompt_tokens = self.input_tokens + if self.completion_tokens is None and self.output_tokens is not None: + self.completion_tokens = self.output_tokens + if self.total_tokens is None and self.input_tokens is not None and self.output_tokens is not None: + self.total_tokens = self.input_tokens + self.output_tokens + return self + + +class LMOutput(BaseModel): + """One generated candidate in an LM response.""" + + parts: list[LMPart] = Field(default_factory=list) + finish_reason: str | None = None + truncated: bool = False + logprobs: Any | None = None + provider_output: Any | None = Field(default=None, exclude=True) + provider_data: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + @model_validator(mode="before") + @classmethod + def normalize_parts(cls, data: Any) -> Any: + if isinstance(data, dict) and "parts" in data: + data = dict(data) + data["parts"] = [_coerce_part(part) for part in data["parts"]] + return data + + @property + def text(self) -> str | None: + texts = [part.text for part in self.parts if isinstance(part, LMTextPart)] + return "".join(texts) if texts else None + + @property + def reasoning_content(self) -> str | None: + texts = [part.text for part in self.parts if isinstance(part, LMThinkingPart)] + return "".join(texts) if texts else None + + @property + def tool_calls(self) -> list[LMToolCallPart]: + return [part for part in self.parts if isinstance(part, LMToolCallPart)] + + @property + def citations(self) -> list[LMCitationPart]: + return [part for part in self.parts if isinstance(part, LMCitationPart)] + + @property + def images(self) -> list[LMImagePart]: + return [part for part in self.parts if isinstance(part, LMImagePart)] + + @property + def audio(self) -> list[LMAudioPart]: + return [part for part in self.parts if isinstance(part, LMAudioPart)] + + @property + def videos(self) -> list[LMVideoPart]: + return [part for part in self.parts if isinstance(part, LMVideoPart)] + + @property + def documents(self) -> list[LMDocumentPart]: + return [part for part in self.parts if isinstance(part, LMDocumentPart)] + + @property + def binaries(self) -> list[LMBinaryPart]: + return [part for part in self.parts if isinstance(part, LMBinaryPart)] + + @property + def refusal(self) -> str | None: + refusals = [part.text for part in self.parts if isinstance(part, LMRefusalPart)] + return "".join(refusals) if refusals else None + + def to_value(self) -> Any: + values = [_part_to_value(part) for part in self.parts] + values = [value for value in values if value is not None] + if len(values) == 1 and isinstance(values[0], str) and self.logprobs is None: + return values[0] + return values + + def to_output_dict(self) -> dict[str, Any]: + data: dict[str, Any] = {"text": self.text} + if self.reasoning_content is not None: + data["reasoning_content"] = self.reasoning_content + if self.tool_calls: + data["tool_calls"] = [_tool_call_to_provider_dict(call) for call in self.tool_calls] + if self.citations: + data["citations"] = [citation.model_dump(exclude_none=True) for citation in self.citations] + if self.logprobs is not None: + data["logprobs"] = self.logprobs + return data + + +class LMResponse(BaseModel): + """The normalized result of one LM request.""" + + model: str | None = None + outputs: list[LMOutput] = Field(min_length=1) + usage: LMUsage | dict[str, Any] | None = None + cost: float | None = None + cache_hit: bool = False + response_id: str | None = None + provider_response: Any | None = Field(default=None, exclude=True) + provider_data: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + @model_validator(mode="before") + @classmethod + def normalize_usage(cls, data: Any) -> Any: + if isinstance(data, dict) and isinstance(data.get("usage"), dict): + data = dict(data) + data["usage"] = LMUsage(**data["usage"]) + return data + + @classmethod + def from_text( + cls, + text: str, + *, + model: str | None = None, + usage: LMUsage | dict[str, Any] | None = None, + cost: float | None = None, + cache_hit: bool = False, + **kwargs: Any, + ) -> LMResponse: + return cls( + model=model, + outputs=[LMOutput(parts=[LMTextPart(text=text)])], + usage=usage, + cost=cost, + cache_hit=cache_hit, + **kwargs, + ) + + def __iter__(self): + return iter(self.to_values()) + + def __getitem__(self, index: int) -> Any: + return self.to_values()[index] + + def __len__(self) -> int: + return len(self.outputs) + + @property + def output(self) -> LMOutput: + return self.outputs[0] + + @property + def parts(self) -> list[LMPart]: + return self.output.parts + + @property + def text(self) -> str | None: + return self.output.text + + @property + def reasoning_content(self) -> str | None: + return self.output.reasoning_content + + @property + def tool_calls(self) -> list[LMToolCallPart]: + return self.output.tool_calls + + @property + def citations(self) -> list[LMCitationPart]: + return self.output.citations + + @property + def images(self) -> list[LMImagePart]: + return self.output.images + + @property + def audio(self) -> list[LMAudioPart]: + return self.output.audio + + @property + def videos(self) -> list[LMVideoPart]: + return self.output.videos + + @property + def documents(self) -> list[LMDocumentPart]: + return self.output.documents + + @property + def binaries(self) -> list[LMBinaryPart]: + return self.output.binaries + + def to_values(self) -> list[Any]: + return [output.to_value() for output in self.outputs] + + def to_outputs(self) -> list[Any]: + outputs: list[Any] = [] + for output in self.outputs: + if _requires_output_dict(output): + outputs.append(output.to_output_dict()) + else: + outputs.append(output.to_value()) + return outputs + + def to_legacy_outputs(self) -> list[Any]: + return self.to_outputs() + + def usage_as_dict(self) -> dict[str, Any]: + if self.usage is None: + return {} + if isinstance(self.usage, LMUsage): + return self.usage.model_dump(exclude_none=True) + return dict(self.usage) + + +class LMHistoryEntry(BaseModel, Mapping[str, Any]): + """A typed history record that can be read like a dictionary. + + Store the canonical request and response, then derive legacy convenience + fields such as `outputs`, `usage`, `messages`, and `kwargs` on demand. + Because this class implements `Mapping`, existing history code can keep + using `entry["messages"]`, `entry.get("prompt")`, `entry.items()`, and + `dict(entry)`. + """ + + request: LMRequest + response: LMResponse + timestamp: str + uuid: str + model_type: str | None = None + + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") + + @model_validator(mode="before") + @classmethod + def drop_derived_fields(cls, data: Any) -> Any: + if isinstance(data, dict): + data = dict(data) + for key in _HISTORY_DERIVED_KEYS: + data.pop(key, None) + return data + + @property + def outputs(self) -> list[Any]: + return self.response.to_outputs() + + @property + def usage(self) -> dict[str, Any]: + return self.response.usage_as_dict() + + @property + def cost(self) -> float | None: + return self.response.cost + + @property + def model(self) -> str: + return self.request.model + + @property + def prompt(self) -> str | None: + return _history_request_prompt(self.request) + + @property + def messages(self) -> list[dict[str, Any]] | None: + return _history_request_messages_as_openai(self.request) + + @property + def kwargs(self) -> dict[str, Any]: + return _history_request_kwargs(self.request) + + @property + def response_model(self) -> str | None: + return self.response.model + + def __getitem__(self, key: str) -> Any: + return self._mapping()[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._mapping()) + + def __len__(self) -> int: + return len(self._mapping()) + + def __repr__(self) -> str: + formatted = pformat(self._essential_mapping(), width=100, sort_dicts=False) + return f"LMHistoryEntry(\n{formatted}\n)" + + def __str__(self) -> str: + return repr(self) + + def to_dict(self, *, mode: str = "python", exclude_none: bool = False, **kwargs: Any) -> dict[str, Any]: + """Return this history entry as a plain dictionary.""" + if kwargs: + return self.model_dump(mode=mode, exclude_none=exclude_none, **kwargs) + data = self._mapping() + if mode != "python": + data = json.loads(json.dumps(data, default=_json_default)) + if exclude_none: + data = {key: value for key, value in data.items() if value is not None} + return data + + def _essential_mapping(self) -> dict[str, Any]: + data = self.model_dump(mode="python", exclude_none=True) + data.update(self.model_extra or {}) + return data + + def _mapping(self) -> dict[str, Any]: + data = self._essential_mapping() + data.update({key: getattr(self, key) for key in _HISTORY_DERIVED_KEYS}) + return {key: value for key, value in data.items() if value is not None} + + +_HISTORY_DERIVED_KEYS = ( + "outputs", + "usage", + "cost", + "model", + "prompt", + "messages", + "kwargs", + "response_model", +) + + +def _json_default(value: Any) -> Any: + if hasattr(value, "model_dump"): + return value.model_dump(mode="json", exclude_none=True) + return str(value) + + +class LMDelta(BaseModel): + """Base class for streamed content deltas.""" + + type: str + + +class LMTextDelta(LMDelta): + type: Literal["text_delta"] = "text_delta" + text: str + + +class LMThinkingDelta(LMDelta): + type: Literal["thinking_delta"] = "thinking_delta" + text: str + + +class LMToolCallDelta(LMDelta): + type: Literal["tool_call_delta"] = "tool_call_delta" + id: str | None = None + name: str | None = None + args_delta: str | None = None + + +class LMCitationDelta(LMDelta): + type: Literal["citation_delta"] = "citation_delta" + citation: LMCitationPart + + +class LMImageDelta(LMDelta): + type: Literal["image_delta"] = "image_delta" + image: LMImagePart + + +class LMAudioDelta(LMDelta): + type: Literal["audio_delta"] = "audio_delta" + audio: LMAudioPart + + +LMAnyDelta = Annotated[ + LMTextDelta | LMThinkingDelta | LMToolCallDelta | LMCitationDelta | LMImageDelta | LMAudioDelta, + Field(discriminator="type"), +] + + +class LMStreamEvent(BaseModel): + """Base class for normalized LM stream events.""" + + type: str + + +class LMStreamStartEvent(LMStreamEvent): + type: Literal["start"] = "start" + model: str | None = None + + +class LMStreamDeltaEvent(LMStreamEvent): + type: Literal["delta"] = "delta" + output_index: int = Field(default=0, ge=0) + part_index: int = Field(ge=0) + delta: LMAnyDelta + + +class LMStreamOutputEndEvent(LMStreamEvent): + type: Literal["output_end"] = "output_end" + output_index: int = Field(default=0, ge=0) + finish_reason: str | None = None + truncated: bool = False + + +class LMStreamEndEvent(LMStreamEvent): + type: Literal["end"] = "end" + usage: LMUsage | dict[str, Any] | None = None + cost: float | None = None + response: LMResponse | None = None + + +class LMStreamErrorEvent(LMStreamEvent): + type: Literal["error"] = "error" + error: Exception + + model_config = ConfigDict(arbitrary_types_allowed=True) + + +class LMOutputBuilder: + """Assemble streamed LM events into a final `LMResponse`.""" + + def __init__(self): + self.model: str | None = None + self._parts: dict[int, list[LMPart | None]] = {} + self._finish_reasons: dict[int, str | None] = {} + self._truncated: dict[int, bool] = {} + + def apply(self, event: LMStreamEvent) -> LMResponse | None: + if isinstance(event, LMStreamStartEvent): + self.model = event.model + return None + if isinstance(event, LMStreamDeltaEvent): + self._apply_delta(event) + return None + if isinstance(event, LMStreamOutputEndEvent): + self._finish_reasons[event.output_index] = event.finish_reason + self._truncated[event.output_index] = event.truncated + return None + if isinstance(event, LMStreamEndEvent): + if event.response is not None: + return event.response + return self.to_response(usage=event.usage, cost=event.cost) + if isinstance(event, LMStreamErrorEvent): + raise event.error + return None + + def to_response(self, *, usage: LMUsage | dict[str, Any] | None = None, cost: float | None = None) -> LMResponse: + output_indices = set(self._parts) | set(self._finish_reasons) | set(self._truncated) + if not output_indices: + output_indices = {0} + max_index = max(output_indices) + expected_indices = set(range(max_index + 1)) + if output_indices != expected_indices: + missing = sorted(expected_indices - output_indices) + raise ValueError(f"Stream output indices must be contiguous from 0; missing indices: {missing}.") + + outputs = [] + for output_index in range(max_index + 1): + part_buffer = self._parts.get(output_index, []) + missing_part_indices = [index for index, part in enumerate(part_buffer) if part is None] + if missing_part_indices: + raise ValueError( + f"Stream part indices for output {output_index} must be contiguous; " + f"missing indices: {missing_part_indices}." + ) + parts = [_finalize_stream_part(part) for part in part_buffer] + outputs.append( + LMOutput( + parts=parts, + finish_reason=self._finish_reasons.get(output_index), + truncated=self._truncated.get(output_index, False), + ) + ) + return LMResponse(model=self.model, outputs=outputs, usage=usage, cost=cost) + + def _apply_delta(self, event: LMStreamDeltaEvent) -> None: + parts = self._parts.setdefault(event.output_index, []) + while len(parts) <= event.part_index: + parts.append(None) + + current = parts[event.part_index] + delta = event.delta + if isinstance(delta, LMThinkingDelta): + if current is not None and not isinstance(current, LMThinkingPart): + raise ValueError("Cannot apply thinking delta to a non-thinking stream part.") + text = (current.text if isinstance(current, LMThinkingPart) else "") + delta.text + parts[event.part_index] = LMThinkingPart(text=text) + elif isinstance(delta, LMTextDelta): + if current is not None and not isinstance(current, LMTextPart): + raise ValueError("Cannot apply text delta to a non-text stream part.") + text = (current.text if isinstance(current, LMTextPart) else "") + delta.text + parts[event.part_index] = LMTextPart(text=text) + elif isinstance(delta, LMToolCallDelta): + if current is not None and not isinstance(current, LMToolCallPart): + raise ValueError("Cannot apply tool-call delta to a non-tool-call stream part.") + buffer = "" + if isinstance(current, LMToolCallPart): + buffer = current.provider_data.get("args_buffer", "") + buffer += delta.args_delta or "" + args = _parse_json_object(buffer) + parts[event.part_index] = LMToolCallPart( + id=delta.id if delta.id is not None else getattr(current, "id", None), + name=delta.name if delta.name is not None else getattr(current, "name", ""), + args=args, + provider_data={"args_buffer": buffer}, + ) + elif isinstance(delta, LMCitationDelta): + if current is not None and not isinstance(current, LMCitationPart): + raise ValueError("Cannot apply citation delta to a different stream part type.") + parts[event.part_index] = delta.citation + elif isinstance(delta, LMImageDelta): + if current is not None and not isinstance(current, LMImagePart): + raise ValueError("Cannot apply image delta to a different stream part type.") + parts[event.part_index] = delta.image + elif isinstance(delta, LMAudioDelta): + if current is not None and not isinstance(current, LMAudioPart): + raise ValueError("Cannot apply audio delta to a different stream part type.") + parts[event.part_index] = delta.audio + + +class LMStream: + """Synchronous LM stream with a final `LMResponse` result.""" + + def __init__( + self, + *, + request: LMRequest, + events: Iterator[LMStreamEvent], + finalize: Callable[[LMRequest, LMResponse], LMResponse], + ): + self.request = request + self._events = events + self._finalize = finalize + self._builder = LMOutputBuilder() + self._result: LMResponse | None = None + + def __iter__(self) -> Iterator[LMStreamEvent]: + for event in self._events: + response = self._builder.apply(event) + if response is not None: + self._result = self._finalize(self.request, response) + yield event + + def result(self) -> LMResponse: + if self._result is None: + raise RuntimeError("Stream has not completed yet.") + return self._result + + +class AsyncLMStream: + """Asynchronous LM stream with a final `LMResponse` result.""" + + def __init__( + self, + *, + request: LMRequest, + events: AsyncIterator[LMStreamEvent], + finalize: Callable[[LMRequest, LMResponse], LMResponse], + ): + self.request = request + self._events = events + self._finalize = finalize + self._builder = LMOutputBuilder() + self._result: LMResponse | None = None + + async def __aiter__(self) -> AsyncIterator[LMStreamEvent]: + async for event in self._events: + response = self._builder.apply(event) + if response is not None: + self._result = self._finalize(self.request, response) + yield event + + def result(self) -> LMResponse: + if self._result is None: + raise RuntimeError("Stream has not completed yet.") + return self._result + + +def System(*parts: Any, name: str | None = None, metadata: dict[str, Any] | None = None) -> LMMessage: # noqa: N802 + """Create a system message for a direct LM call. + + A system message gives model-level instructions, such as tone, scope, or + formatting rules. Pass text, media parts, or normalized `LMPart` objects; + DSPy stores them as one `LMMessage` with role `"system"`. + + Args: + *parts: Text, DSPy media objects, or normalized LM parts to include in + the message. + name: Optional sender name for providers that support named messages. + metadata: Extra information to keep with the message. + + Returns: + An `LMMessage` that can be passed to `dspy.LanguageModel`, `dspy.LM`, or + `dspy.LMRequest`. + + Examples: + System instruction with a user turn: + + ```python + import dspy + + lm = dspy.LanguageModel(model="test/model") + request = lm.normalize_request( + dspy.System("You are concise."), + dspy.User("What is DSPy?"), + ) + ``` + + See Also: + [`dspy.User`][dspy.User] + [`dspy.Assistant`][dspy.Assistant] + [`dspy.LMRequest`][dspy.LMRequest] + """ + return LMMessage(role="system", parts=[_coerce_part(part) for part in parts], name=name, metadata=metadata or {}) + + +def Developer(*parts: Any, name: str | None = None, metadata: dict[str, Any] | None = None) -> LMMessage: # noqa: N802 + """Create a developer message for a direct LM call. + + A developer message carries instructions that sit between system guidance + and user content. Use it when a provider supports a `"developer"` role and + you want to keep implementation guidance separate from the user's request. + + Args: + *parts: Text, DSPy media objects, or normalized LM parts to include in + the message. + name: Optional sender name for providers that support named messages. + metadata: Extra information to keep with the message. + + Returns: + An `LMMessage` with role `"developer"`. + + Examples: + Add house-style instructions: + + ```python + import dspy + + request = dspy.LMRequest.from_call( + model="openai/gpt-4o-mini", + items=( + dspy.System("You are a technical editor."), + dspy.Developer("Prefer short examples."), + dspy.User("Explain callbacks."), + ), + ) + ``` + + See Also: + [`dspy.System`][dspy.System] + [`dspy.User`][dspy.User] + """ + return LMMessage(role="developer", parts=[_coerce_part(part) for part in parts], name=name, metadata=metadata or {}) + + +def User(*parts: Any, name: str | None = None, metadata: dict[str, Any] | None = None) -> LMMessage: # noqa: N802 + """Create a user message for a direct LM call. + + A user message contains the request or data you want the model to answer. + Pass plain text for simple prompts, or mix text with images, audio, documents, + binary attachments, and normalized LM parts for multimodal calls. + + Args: + *parts: Text, DSPy media objects, or normalized LM parts to include in + the message. + name: Optional sender name for providers that support named messages. + metadata: Extra information to keep with the message. + + Returns: + An `LMMessage` with role `"user"`. + + Examples: + Multi-turn LM call: + + ```python + import dspy + + lm = dspy.LM("openai/gpt-4o-mini") + response = lm( + dspy.User("What is DSPy?"), + dspy.Assistant("DSPy is a framework for programming LM pipelines."), + dspy.User("Say that in five words."), + ) + ``` + + Multi-turn call with media: + + ```python + import dspy + + lm = dspy.LM("openai/gpt-4o-mini") + response = lm( + dspy.System("Answer in one sentence."), + dspy.User( + "Describe this image.", + dspy.Image("https://example.com/dog.png"), + ), + ) + ``` + + For a single user turn, pass the parts directly to `lm(...)` instead: + + ```python + response = lm("Describe this image.", dspy.Image("https://example.com/dog.png")) + ``` + + Explicit `LMRequest` for custom LM authors and advanced users: + + ```python + import dspy + + lm = dspy.LM("openai/gpt-4o-mini") + request = dspy.LMRequest( + model="openai/gpt-4o-mini", + messages=[ + dspy.System("You are concise."), + dspy.User( + "Describe this image.", + dspy.Image("https://example.com/dog.png"), + ), + ], + config=dspy.LMConfig(temperature=0.2, max_tokens=200), + ) + + response = lm(request) + ``` + + See Also: + [`dspy.System`][dspy.System] + [`dspy.Assistant`][dspy.Assistant] + [`dspy.ToolResult`][dspy.ToolResult] + """ + return LMMessage(role="user", parts=[_coerce_part(part) for part in parts], name=name, metadata=metadata or {}) + + +def Assistant(*parts: Any, name: str | None = None, metadata: dict[str, Any] | None = None) -> LMMessage: # noqa: N802 + """Create an assistant message for a direct LM call. + + An assistant message represents a previous model response. Use it when you + build a multi-turn request by hand, or when you send tool calls back to the + model before adding tool results. + + Args: + *parts: Text, tool calls, citations, media parts, or other normalized LM + parts from an assistant turn. + name: Optional sender name for providers that support named messages. + metadata: Extra information to keep with the message. + + Returns: + An `LMMessage` with role `"assistant"`. + + Examples: + Continue a conversation: + + ```python + import dspy + + request = dspy.LMRequest.from_call( + model="openai/gpt-4o-mini", + items=( + dspy.User("What is DSPy?"), + dspy.Assistant("DSPy is a framework for programming LM pipelines."), + dspy.User("Say that in five words."), + ), + ) + ``` + + See Also: + [`dspy.User`][dspy.User] + [`dspy.ToolCall`][dspy.ToolCall] + [`dspy.ToolResult`][dspy.ToolResult] + """ + return LMMessage(role="assistant", parts=[_coerce_part(part) for part in parts], name=name, metadata=metadata or {}) + + +ToolCall = LMToolCallPart +"""Create a tool-call part for an assistant message. + +`ToolCall` is an alias for `LMToolCallPart`. Use it inside `dspy.Assistant(...)` +when you want to include a model-requested tool call in a normalized +conversation. +""" + + +def ToolResult( # noqa: N802 + *parts: Any, + call_id: str | None = None, + name: str | None = None, + content: Any | None = None, + is_error: bool = False, +) -> LMMessage: + """Create a tool-result message for a direct LM call. + + A tool-result message sends the output of a tool back to the model. Pass the + returned text or media as `*parts` or with `content=...`, and include the + `call_id` from the matching assistant tool call when the provider uses call + IDs. + + Args: + *parts: Text, DSPy media objects, or normalized LM parts returned by the + tool. If you pass one `LMToolResultPart`, DSPy uses it directly. + call_id: Identifier of the assistant tool call this result answers. + name: Tool name associated with the result. + content: Optional tool output passed by keyword. Use this when adapting + OpenAI-style code that stores the result under `content`. + is_error: Whether this result represents a failed tool execution. + + Returns: + An `LMMessage` with role `"tool"` and one `LMToolResultPart`. + + Examples: + Send a weather result back to the model: + + ```python + import dspy + + messages = [ + dspy.User("What is the weather in Paris?"), + dspy.Assistant( + dspy.ToolCall( + id="call_1", + name="get_weather", + args={"location": "Paris"}, + ) + ), + dspy.ToolResult( + '{"temperature": "22", "unit": "celsius"}', + call_id="call_1", + name="get_weather", + ), + dspy.User("Summarize the result."), + ] + ``` + + See Also: + [`dspy.ToolCall`][dspy.ToolCall] + [`dspy.Assistant`][dspy.Assistant] + [`dspy.Tool`][dspy.Tool] + """ + if content is not None: + if parts: + raise TypeError("Pass tool output either as positional parts or as `content=...`, not both.") + parts = tuple(content if isinstance(content, list) else [content]) + + if len(parts) == 1 and isinstance(parts[0], LMToolResultPart): + result = parts[0] + else: + result = LMToolResultPart( + call_id=call_id, + name=name, + content=[_coerce_part(part) for part in parts], + is_error=is_error, + ) + return LMMessage(role="tool", parts=[result]) + + +def _history_request_prompt(request: LMRequest) -> str | None: + if len(request.messages) != 1: + return None + message = request.messages[0] + if message.role != "user" or len(message.parts) != 1: + return None + part = message.parts[0] + return part.text if isinstance(part, LMTextPart) else None + + +def _history_request_messages_as_openai(request: LMRequest) -> list[dict[str, Any]]: + messages = [] + for message in request.messages: + if message.role == "assistant": + tool_calls = [part for part in message.parts if isinstance(part, LMToolCallPart)] + content_parts = [part for part in message.parts if not isinstance(part, LMToolCallPart)] + item: dict[str, Any] = { + "role": "assistant", + "content": _history_message_parts_as_openai_content(content_parts) if content_parts else None, + } + if tool_calls: + item["tool_calls"] = [_history_tool_call_as_openai(call) for call in tool_calls] + elif message.role == "tool" and len(message.parts) == 1 and isinstance(message.parts[0], LMToolResultPart): + result = message.parts[0] + item = {"role": "tool", "content": _history_tool_result_content(result)} + if result.call_id is not None: + item["tool_call_id"] = result.call_id + if result.name is not None: + item["name"] = result.name + else: + item = { + "role": message.role, + "content": _history_message_parts_as_openai_content(message.parts), + } + if message.name is not None and "name" not in item: + item["name"] = message.name + messages.append(item) + return messages + + +def _history_tool_call_as_openai(call: LMToolCallPart) -> dict[str, Any]: + data: dict[str, Any] = { + "type": "function", + "function": { + "name": call.name, + "arguments": json.dumps(call.args), + }, + } + if call.id is not None: + data["id"] = call.id + return data + + +def _history_tool_result_content(result: LMToolResultPart) -> str: + chunks = [] + for part in result.content: + if isinstance(part, LMTextPart): + chunks.append(part.text) + else: + chunks.append(json.dumps(part.model_dump(mode="json", exclude_none=True), ensure_ascii=False)) + return "".join(chunks) + + +def _history_message_parts_as_openai_content(parts: list[LMPart]) -> str | list[dict[str, Any]]: + if len(parts) == 1 and isinstance(parts[0], LMTextPart): + return parts[0].text + return [_history_part_as_openai_content(part) for part in parts] + + +def _history_part_as_openai_content(part: LMPart) -> dict[str, Any]: + if isinstance(part, LMTextPart): + return {"type": "text", "text": part.text} + if isinstance(part, LMImagePart): + return {"type": "image_url", "image_url": {"url": _history_part_source(part)}} + if isinstance(part, LMAudioPart): + input_audio = {"format": _history_media_format(part.media_type)} + if part.data is not None: + if part.data.startswith("data:"): + media_type, data = _split_data_uri(part.data) + input_audio["format"] = _history_media_format(media_type) + input_audio["data"] = data + else: + input_audio["data"] = part.data + elif part.url is not None: + input_audio["url"] = part.url + elif part.file_id is not None: + input_audio["file_id"] = part.file_id + elif part.path is not None: + input_audio["path"] = part.path + return { + "type": "input_audio", + "input_audio": input_audio, + } + if isinstance(part, LMVideoPart): + video = {"media_type": part.media_type} + if part.data is not None: + video["data"] = _history_part_source(part) + elif part.url is not None: + video["url"] = part.url + elif part.file_id is not None: + video["file_id"] = part.file_id + elif part.path is not None: + video["path"] = part.path + return {"type": "video", "video": video} + if isinstance(part, LMDocumentPart): + data = {"type": "document"} + if part.source is not None: + data["source"] = part.source + else: + data["source"] = _history_part_source(part) + data["media_type"] = part.media_type + if part.citations: + data["citations"] = part.citations + if part.title is not None: + data["title"] = part.title + if part.context is not None: + data["context"] = part.context + return data + if isinstance(part, LMBinaryPart): + return { + "type": "binary", + "binary": { + key: value + for key, value in { + "data": _history_part_source(part), + "file_id": part.file_id, + "filename": part.filename, + "media_type": part.media_type, + }.items() + if value is not None + }, + } + return part.model_dump(exclude_none=True) + + +def _history_part_source(part: LMImagePart | LMAudioPart | LMVideoPart | LMDocumentPart | LMBinaryPart) -> str | None: + if part.data is not None: + return part.data if part.data.startswith("data:") else f"data:{part.media_type};base64,{part.data}" + return part.url or part.file_id or part.path + + +def _history_media_format(media_type: str) -> str: + return media_type.split("/", 1)[1] if "/" in media_type else media_type + + +def _history_request_kwargs(request: LMRequest) -> dict[str, Any]: + data = request.config.model_dump(exclude_none=True) + extensions = data.pop("extensions", {}) or {} + return {**extensions, **data} + + +def _validate_one_source(part: Any) -> None: + sources = {name: getattr(part, name) for name in ("data", "url", "file_id", "path") if getattr(part, name) is not None} + class_name = type(part).__name__ + if len(sources) != 1: + raise ValueError(f"{class_name} requires exactly one of data, url, file_id, or path.") + name, value = next(iter(sources.items())) + if isinstance(value, str) and not value: + raise ValueError(f"{class_name}.{name} must be non-empty.") + + +def _coerce_message(value: dict[str, Any] | LMMessage) -> LMMessage: + if isinstance(value, LMMessage): + return value + return LMMessage(**value) + + +def _messages_from_items(items: tuple[Any, ...], *, prompt: str | None = None) -> tuple[list[LMMessage], list[Any]]: + # TODO: Normalize DSPy-specific LM(...) objects in the LM call layer before building LMRequest. + if prompt is not None: + items = (prompt, *items) + if not items: + items = ("",) + + if len(items) == 1 and _is_message_sequence(items[0]): + items = tuple(items[0]) + + if all(isinstance(item, LMMessage) or isinstance(item, LMResponse) for item in items): + messages: list[LMMessage] = [] + for item in items: + if isinstance(item, LMMessage): + messages.append(item) + else: + messages.extend(_messages_from_response(item)) + return messages, [] + + parts = [_coerce_part(item) for item in items] + return [LMMessage(role="user", parts=parts)], [] + + +def _messages_from_response(response: LMResponse) -> list[LMMessage]: + return [LMMessage(role="assistant", parts=output.parts) for output in response.outputs] + + +def _is_message_sequence(value: Any) -> bool: + return isinstance(value, (list, tuple)) and all( + isinstance(item, LMMessage) or isinstance(item, LMResponse) for item in value + ) + + +def _coerce_part(value: Any) -> LMPart: + if isinstance( + value, + ( + LMTextPart, + LMImagePart, + LMAudioPart, + LMVideoPart, + LMDocumentPart, + LMBinaryPart, + LMToolCallPart, + LMToolResultPart, + LMThinkingPart, + LMCitationPart, + LMRefusalPart, + ), + ): + return value + if isinstance(value, str): + return LMTextPart(text=value) + if isinstance(value, dict) and "type" in value: + return pydantic.TypeAdapter(LMPart).validate_python(value) + raise TypeError(f"Cannot convert {type(value)!r} to an LMPart.") + + +def _parts_from_openai_content(content: Any) -> list[LMPart]: + if content is None: + return [] + if isinstance(content, str): + return [LMTextPart(text=content)] + if not isinstance(content, list): + return [_coerce_part(content)] + + parts = [] + for item in content: + item_type = item.get("type") if isinstance(item, dict) else None + if item_type == "text": + parts.append(LMTextPart(text=item.get("text", ""), metadata=item.get("metadata", {}) or {})) + elif item_type == "image_url": + image = item.get("image_url", {}) + if not isinstance(image, dict): + raise TypeError("Image content block must be a mapping.") + url = image.get("url") + if url is None: + raise ValueError("Image content block requires url.") + parts.append(_image_source_to_part(url)) + elif item_type == "input_audio": + audio = item.get("input_audio", {}) + parts.append(_audio_dict_to_part(audio)) + elif item_type == "file": + parts.append(_binary_dict_to_part(item.get("file", {}))) + elif item_type == "document": + parts.append(_document_dict_to_part(item)) + elif item_type == "video": + video = item.get("video", {}) + parts.append(_media_dict_to_video_part(video)) + else: + parts.append(_coerce_part(item)) + return parts + + +def _tool_calls_from_openai(tool_calls: list[Any]) -> list[LMToolCallPart]: + return [_tool_call_from_openai(tool_call) for tool_call in tool_calls] + + +def _tool_call_from_openai(tool_call: Any) -> LMToolCallPart: + if not isinstance(tool_call, Mapping): + part = _coerce_part(tool_call) + if isinstance(part, LMToolCallPart): + return part + raise TypeError(f"Cannot convert {type(tool_call)!r} to an LMToolCallPart.") + + function = tool_call.get("function", {}) + if not isinstance(function, Mapping): + function = {} + + args = function.get("arguments", {}) + if isinstance(args, str): + args = _parse_json_object(args) + elif isinstance(args, Mapping): + args = dict(args) + else: + args = {} + + return LMToolCallPart( + id=tool_call.get("id"), + name=function.get("name") or tool_call.get("name") or "", + args=args, + ) + + +def _image_source_to_part(source: str) -> LMImagePart: + if not isinstance(source, str): + raise TypeError("Image URL must be a string.") + if source.startswith("data:"): + media_type, data = _split_data_uri(source) + return LMImagePart(data=data, media_type=media_type) + media_type = mimetypes.guess_type(urlparse(source).path)[0] or "image/png" + return LMImagePart(url=source, media_type=media_type) + + +def _audio_dict_to_part(audio: dict[str, Any]) -> LMAudioPart: + if not isinstance(audio, dict): + raise TypeError("Audio content block must be a mapping.") + + audio_format = audio.get("format") or "wav" + if not isinstance(audio_format, str): + raise TypeError("Audio format must be a string.") + media_type = audio_format if "/" in audio_format else f"audio/{audio_format}" + + if audio.get("data") is not None: + data = audio["data"] + if isinstance(data, str) and data.startswith("data:"): + media_type, data = _split_data_uri(data) + return LMAudioPart(data=data, media_type=media_type) + if audio.get("url") is not None: + return LMAudioPart(url=audio["url"], media_type=media_type) + if audio.get("file_id") is not None: + return LMAudioPart(file_id=audio["file_id"], media_type=media_type) + if audio.get("path") is not None: + return LMAudioPart(path=audio["path"], media_type=media_type) + raise ValueError("Audio content block requires data, url, file_id, or path.") + + +def _binary_dict_to_part(file: dict[str, Any]) -> LMBinaryPart: + if file.get("file_data") is not None: + media_type, data = _split_data_uri(file["file_data"]) + return LMBinaryPart(data=data, media_type=media_type, filename=file.get("filename")) + if file.get("data") is not None: + media_type, data = _split_data_uri(file["data"]) + return LMBinaryPart(data=data, media_type=media_type, filename=file.get("filename")) + if file.get("file_id") is not None: + return LMBinaryPart(file_id=file["file_id"], filename=file.get("filename")) + raise ValueError("Binary content block requires data, file_data, or file_id.") + + +def _document_dict_to_part(item: dict[str, Any]) -> LMDocumentPart: + common = {"title": item.get("title"), "context": item.get("context")} + media_type = item.get("media_type") or "application/pdf" + for source_key in ("data", "url", "file_id", "path"): + if item.get(source_key) is not None: + return LMDocumentPart(**{source_key: item[source_key]}, media_type=media_type, **common) + + source = item.get("source") + if isinstance(source, dict): + return LMDocumentPart( + source=source, + citations=item.get("citations") or {}, + **common, + ) + if isinstance(source, str): + kwargs = _media_source_kwargs(source, default_media_type=media_type) + return LMDocumentPart(**kwargs, **common) + raise ValueError("Document content block requires source.") + + +def _media_dict_to_video_part(video: dict[str, Any]) -> LMVideoPart: + if video.get("data") is not None: + data = video["data"] + if isinstance(data, str) and data.startswith("data:"): + media_type, data = _split_data_uri(data) + else: + media_type = video.get("media_type") or "video/mp4" + return LMVideoPart(data=data, media_type=media_type) + if video.get("url") is not None: + return LMVideoPart(url=video["url"], media_type=video.get("media_type") or "video/mp4") + if video.get("file_id") is not None: + return LMVideoPart(file_id=video["file_id"], media_type=video.get("media_type") or "video/mp4") + if video.get("path") is not None: + return LMVideoPart(path=video["path"], media_type=video.get("media_type") or "video/mp4") + raise ValueError("Video content block requires data, url, file_id, or path.") + + +def _split_data_uri(value: str) -> tuple[str, str]: + if not value.startswith("data:") or "," not in value: + return "application/octet-stream", value + header, data = value.split(",", 1) + media_type = header.removeprefix("data:").split(";", 1)[0] + return media_type, data + + +def _media_source_kwargs(source: str, *, default_media_type: str) -> dict[str, str]: + if source.startswith("data:"): + media_type, data = _split_data_uri(source) + return {"data": data, "media_type": media_type} + + parsed = urlparse(source) + if parsed.scheme in {"http", "https"}: + media_type = mimetypes.guess_type(parsed.path)[0] or default_media_type + return {"url": source, "media_type": media_type} + + return {"file_id": source, "media_type": default_media_type} + + +def _coerce_tool_spec(tool: Any) -> LMToolSpec: + if isinstance(tool, LMToolSpec): + return tool + if hasattr(tool, "to_lm_tool_spec"): + return tool.to_lm_tool_spec() + if isinstance(tool, dict): + if "function" in tool: + function = tool["function"] + provider_data = {key: value for key, value in tool.items() if key not in {"type", "function"}} + return LMToolSpec( + name=function.get("name"), + description=function.get("description"), + parameters=function.get("parameters", {}), + provider_data=provider_data, + ) + return LMToolSpec(**tool) + raise TypeError(f"Cannot convert {type(tool)!r} to LMToolSpec.") + + +def _requires_output_dict(output: LMOutput) -> bool: + return bool( + output.logprobs is not None or output.reasoning_content is not None or output.tool_calls or output.citations + ) + + +def _part_to_value(part: LMPart) -> Any: + if isinstance(part, LMTextPart): + return part.text + if isinstance(part, LMThinkingPart): + return part + if isinstance(part, LMToolCallPart): + return part + if isinstance(part, LMRefusalPart): + return part.text + return part + + +def _finalize_stream_part(part: LMPart) -> LMPart: + if isinstance(part, LMToolCallPart) and "args_buffer" in part.provider_data: + return part.model_copy(update={"args": _parse_json_object_strict(part.provider_data["args_buffer"])}) + return part + + +def _tool_call_to_provider_dict(call: LMToolCallPart) -> dict[str, Any]: + data = { + "type": "function", + "function": { + "name": call.name, + "arguments": json.dumps(call.args), + }, + } + if call.id is not None: + data["id"] = call.id + return data + + +def _parse_json_object(value: str) -> dict[str, Any]: + if not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _parse_json_object_strict(value: str) -> dict[str, Any]: + if not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError("Streamed tool-call arguments must be a JSON object.") from exc + if not isinstance(parsed, dict): + raise ValueError("Streamed tool-call arguments must be a JSON object.") + return parsed diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py index 906ef90ae9..e3ad562a6a 100644 --- a/dspy/predict/__init__.py +++ b/dspy/predict/__init__.py @@ -8,6 +8,7 @@ from dspy.predict.predict import Predict from dspy.predict.program_of_thought import ProgramOfThought from dspy.predict.react import ReAct, Tool +from dspy.predict.reactv2 import ReActV2 from dspy.predict.refine import Refine from dspy.predict.rlm import RLM @@ -21,6 +22,7 @@ "Predict", "ProgramOfThought", "ReAct", + "ReActV2", "Refine", "RLM", "Tool", diff --git a/dspy/predict/avatar/signatures.py b/dspy/predict/avatar/signatures.py index 65d1147521..13ac0cca9e 100644 --- a/dspy/predict/avatar/signatures.py +++ b/dspy/predict/avatar/signatures.py @@ -10,14 +10,11 @@ class Actor(dspy.Signature): Note: You can opt to use no tools and provide the final answer directly. You can also one tool multiple times with different input queries if applicable.""" goal: str = dspy.InputField( - prefix="Goal:", desc="Task to be accomplished.", ) tools: list[str] = dspy.InputField( - prefix="Tools:", desc="list of tools to use", ) action_1: Action = dspy.OutputField( - prefix="Action 1:", desc="1st action to take.", ) diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py index bd790a6f32..6212806bf7 100644 --- a/dspy/predict/predict.py +++ b/dspy/predict/predict.py @@ -1,3 +1,4 @@ +import importlib import logging import random import types @@ -7,6 +8,7 @@ from pydantic_core import PydanticUndefined from dspy.adapters.chat_adapter import ChatAdapter +from dspy.adapters.types.history import History from dspy.clients.base_lm import BaseLM from dspy.clients.lm import LM from dspy.dsp.utils.settings import settings @@ -20,6 +22,7 @@ logger = logging.getLogger(__name__) UNSAFE_LM_STATE_KEYS = {"api_base", "base_url", "model_list"} +_LM_CLASS_STATE_KEY = "_dspy_lm_class" def _sanitize_lm_state(lm_state: dict, allow_unsafe_lm_state: bool) -> dict: @@ -40,6 +43,18 @@ def _sanitize_lm_state(lm_state: dict, allow_unsafe_lm_state: bool) -> dict: return sanitized_lm_state +def _field_is_present_in_history(signature: type[Signature], inputs: dict[str, Any], field_name: str) -> bool: + for history_name, field in signature.input_fields.items(): + if field.annotation != History: + continue + history = inputs.get(history_name) + if not isinstance(history, History): + continue + if any(isinstance(message, dict) and field_name in message for message in history.messages): + return True + return False + + class Predict(Module, Parameter): """Basic DSPy module that maps inputs to outputs using a language model. @@ -86,7 +101,7 @@ def dump_state(self, json_mode=True): state["demos"].append(demo.toDict()) state["signature"] = self.signature.dump_state() - state["lm"] = self.lm.dump_state() if self.lm else None + state["lm"] = _dump_lm_state(self.lm) if self.lm else None return state def load_state(self, state: dict, *, allow_unsafe_lm_state: bool = False) -> "Predict": @@ -108,7 +123,7 @@ def load_state(self, state: dict, *, allow_unsafe_lm_state: bool = False) -> "Pr self.signature = self.signature.load_state(state["signature"]) sanitized_lm_state = _sanitize_lm_state(state["lm"], allow_unsafe_lm_state) if state["lm"] else None - self.lm = LM(**sanitized_lm_state) if sanitized_lm_state else None + self.lm = _load_lm_from_state(sanitized_lm_state) if sanitized_lm_state else None if "extended_signature" in state: # legacy, up to and including 2.5, for CoT. raise NotImplementedError("Loading extended_signature is no longer supported in DSPy 2.6+") @@ -216,11 +231,13 @@ def _forward_preprocess(self, **kwargs): if not all(k in kwargs for k in signature.input_fields): present = [k for k in signature.input_fields if k in kwargs] missing = [k for k in signature.input_fields if k not in kwargs] - logger.warning( - "Not all input fields were provided to module. Present: %s. Missing: %s.", - present, - missing, - ) + missing = [field_name for field_name in missing if not _field_is_present_in_history(signature, kwargs, field_name)] + if missing: + logger.warning( + "Not all input fields were provided to module. Present: %s. Missing: %s.", + present, + missing, + ) return lm, config, signature, demos, kwargs def _forward_postprocess(self, completions, signature, **kwargs): @@ -276,6 +293,57 @@ def get_config(self): def __repr__(self): return f"{self.__class__.__name__}({self.signature})" + +def _dump_lm_state(lm: BaseLM) -> dict[str, Any]: + """Return LM state with enough type information to reconstruct it.""" + lm_state = dict(lm.dump_state()) + lm_state.setdefault(_LM_CLASS_STATE_KEY, f"{type(lm).__module__}.{type(lm).__qualname__}") + return lm_state + + +def _load_lm_from_state(lm_state: dict[str, Any]) -> BaseLM: + """Reconstruct a legacy or custom BaseLM from serialized state. + + Older saved programs did not record the concrete LM class, so they are + loaded through the legacy LiteLLM-backed `dspy.clients.lm.LM` constructor. + Newer states record the concrete class so `Predict` can round-trip any + importable `BaseLM` subclass. + """ + lm_state = dict(lm_state) + class_path = lm_state.pop(_LM_CLASS_STATE_KEY, None) + if class_path is None: + return LM(**lm_state) + + lm_cls = _import_lm_class(class_path) + if not issubclass(lm_cls, BaseLM): + raise TypeError(f"Serialized LM class `{class_path}` must be a subclass of dspy.BaseLM.") + + load_state = getattr(lm_cls, "load_state", None) + if callable(load_state): + return load_state(lm_state) + return lm_cls(**lm_state) + + +def _import_lm_class(class_path: str) -> type: + try: + module_name, class_name = class_path.rsplit(".", 1) + except ValueError as exc: + raise ValueError(f"Invalid serialized LM class path: {class_path!r}") from exc + + try: + module = importlib.import_module(module_name) + lm_cls = getattr(module, class_name) + except (ImportError, AttributeError) as exc: + raise ImportError( + f"Could not import serialized LM class `{class_path}`. Ensure the class is importable, " + "or load an older state without the concrete LM class marker." + ) from exc + + if not isinstance(lm_cls, type): + raise TypeError(f"Serialized LM class `{class_path}` did not resolve to a class.") + return lm_cls + + def _get_type_name(type_annotation) -> str: """Helper method to get the name for a type annotation.""" @@ -354,7 +422,7 @@ def _check_type(value: Any, expected: type) -> bool: return all(_check_type(item, args[0]) for item in value) if len(value) != len(args): return False - return all(_check_type(item, arg) for item, arg in zip(value, args)) + return all(_check_type(item, arg) for item, arg in zip(value, args, strict=True)) return True # set / frozenset diff --git a/dspy/predict/reactv2.py b/dspy/predict/reactv2.py new file mode 100644 index 0000000000..55fddfb311 --- /dev/null +++ b/dspy/predict/reactv2.py @@ -0,0 +1,258 @@ +import logging +import traceback +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable + +import dspy +from dspy.adapters.types.history import History +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls +from dspy.primitives.module import Module +from dspy.signatures.signature import ensure_signature +from dspy.utils.exceptions import AdapterParseError, ContextWindowExceededError + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from dspy.signatures.signature import Signature + +_ANNOTATION_TO_JSON_TYPE = { + str: "string", + int: "integer", + float: "number", + bool: "boolean", + list: "array", +} +_RECOVERABLE_FORCED_SUBMIT_ERRORS = (AdapterParseError, ContextWindowExceededError, ValueError) + + +@dataclass(frozen=True) +class ToolExecutionResult: + value: object + is_error: bool = False + + +def _build_submit_tool(signature: type["Signature"]) -> Tool: + outputs = ", ".join([f"`{k}`" for k in signature.output_fields]) + output_args = {} + output_arg_types = {} + for name, field in signature.output_fields.items(): + annotation = getattr(field, "annotation", str) + # TODO(reactv2): Derive full JSON schema for complex output annotations instead of this primitive map. + output_args[name] = {"type": _ANNOTATION_TO_JSON_TYPE.get(annotation, "string")} + output_arg_types[name] = annotation + + return Tool( + func=lambda **kwargs: kwargs, + name="submit", + desc=f"Call this tool to end the task and return your final answer. Takes: {outputs}.", + args=output_args, + arg_types=output_arg_types, + ) + + +def _optional_input_fields(signature: type["Signature"]) -> dict[str, object]: + fields = deepcopy(signature.input_fields) + for field in fields.values(): + field.default = None + return fields + + +class ReActV2(Module): + def __init__(self, signature: type["Signature"] | str, tools: list[Callable], max_iters: int = 20): + super().__init__() + self.signature = signature = ensure_signature(signature) + self.max_iters = max_iters + + tools = [tool if isinstance(tool, Tool) else Tool(tool) for tool in tools] + self.tools = {tool.name: tool for tool in tools} + self.tools["submit"] = _build_submit_tool(signature) + + react_signature = ( + dspy.Signature(_optional_input_fields(signature), self._build_instructions()) + .append("history", dspy.InputField(), type_=dspy.History) + .append("tools", dspy.InputField(), type_=list[dspy.Tool]) + .append( + "tool_call_results", + dspy.InputField(default=None), + type_=dspy.ToolCallResults | None, + ) + .append("next_thought", dspy.OutputField(), type_=dspy.Reasoning) + .append("tool_calls", dspy.OutputField(), type_=dspy.ToolCalls) + ) + extract_signature = dspy.Signature( + {**signature.input_fields, **signature.output_fields}, + signature.instructions, + ).append( + "trajectory", + dspy.InputField(desc="The agent's history of thoughts, actions, and tool call results"), + type_=str, + ) + self.react = dspy.Predict(react_signature) + self.extract = dspy.ChainOfThought(extract_signature) + + def _build_instructions(self) -> str: + inputs = ", ".join([f"`{k}`" for k in self.signature.input_fields]) + outputs = ", ".join([f"`{k}`" for k in self.signature.output_fields]) + instructions = [f"{self.signature.instructions}\n"] if self.signature.instructions else [] + instructions.extend( + [ + f"You are an Agent. Given {inputs}, use tools to produce {outputs}.", + "Each turn: think, then call one or more tools. After each tool call you receive a tool result.", + "When you have enough information to answer, call `submit` to finish.", + "\nAvailable tools:\n", + ] + ) + instructions.extend(f"({idx + 1}) {tool}" for idx, tool in enumerate(self.tools.values())) + return "\n".join(instructions) + + def forward(self, **input_args): + history = input_args.pop("history", dspy.History(messages=[])) + max_iters = input_args.pop("max_iters", self.max_iters) + tool_list = list(self.tools.values()) + pending_inputs = dict(input_args) + + break_reason = None + for idx in range(max_iters): + try: + pred = self.react(history=history, tools=tool_list, **pending_inputs) + except ContextWindowExceededError: + history.compact_if_needed() + try: + pred = self.react(history=history, tools=tool_list, **pending_inputs) + except ContextWindowExceededError: + logger.warning("Context window exceeded after compaction, ending loop.") + break_reason = "context_overflow" + break + except (AdapterParseError, ValueError) as err: + logger.warning(f"Agent iteration {idx} failed: {_fmt_exc(err)}") + break_reason = "parse_error" + break + + if pred.tool_calls is None or not pred.tool_calls.tool_calls: + logger.warning("Agent returned no tool calls, ending loop.") + break_reason = "no_tool_calls" + break + + tool_calls = pred.tool_calls.with_call_ids(f"call_{len(history.messages)}") + tool_results = [self._execute_tool_call(tool_call) for tool_call in tool_calls.tool_calls] + self._append_tool_turn( + history, + inputs=pending_inputs, + next_thought=pred.next_thought, + tool_calls=tool_calls, + tool_results=tool_results, + ) + pending_inputs = {} + + for tool_call, tool_result in zip(tool_calls.tool_calls, tool_results, strict=True): + if tool_call.name == "submit" and not tool_result.is_error: + history.messages[-1].update(tool_result.value) + return dspy.Prediction(history=history, termination_reason="submit", **tool_result.value) + + return self._forced_submit(history, pending_inputs, break_reason=break_reason) + + def _forced_submit(self, history: History, input_args: dict[str, object], break_reason: str | None = None): + tool_list = list(self.tools.values()) + adapter = dspy.settings.adapter or dspy.ChatAdapter() + call_config = adapter.force_tool_call_config("submit") + + try: + pred = self.react(history=history, tools=tool_list, config=call_config, **input_args) + except _RECOVERABLE_FORCED_SUBMIT_ERRORS as err: + logger.debug(f"Forced submit tier 1 (react) failed: {_fmt_exc(err)}") + pred = None + + if pred and pred.tool_calls and pred.tool_calls.tool_calls: + for tool_call in pred.tool_calls.tool_calls: + if tool_call.name != "submit": + continue + try: + result = self.tools["submit"](**tool_call.args) + except ValueError as err: + logger.debug(f"Forced submit tool execution failed: {_fmt_exc(err)}") + continue + + tool_calls = ToolCalls(tool_calls=[tool_call]).with_call_ids(f"call_{len(history.messages)}") + self._append_tool_turn( + history, + inputs=input_args, + next_thought=pred.next_thought, + tool_calls=tool_calls, + tool_results=[ToolExecutionResult(value=result, is_error=False)], + ) + try: + history.messages[-1].update(result) + return dspy.Prediction(history=history, termination_reason="forced_submit", **result) + except TypeError as err: + logger.debug(f"Forced submit result was not a valid output mapping: {_fmt_exc(err)}") + + try: + trajectory_text = self._render_history_as_text(history) + extract = self.extract(trajectory=trajectory_text, **input_args) + result = {k: getattr(extract, k) for k in self.signature.output_fields if hasattr(extract, k)} + if any(value is not None for value in result.values()): + history.append(result) + return dspy.Prediction(history=history, termination_reason="extract", **result) + except _RECOVERABLE_FORCED_SUBMIT_ERRORS as err: + logger.debug(f"Forced submit tier 2 (extract) failed: {_fmt_exc(err)}") + + return dspy.Prediction(history=history, termination_reason=break_reason or "failed") + + def _execute_tool_call(self, tool_call: ToolCalls.ToolCall) -> ToolExecutionResult: + tool = self.tools.get(tool_call.name) + if tool is None: + return ToolExecutionResult(value=f"Unknown tool: {tool_call.name}", is_error=True) + try: + return ToolExecutionResult(value=tool(**tool_call.args), is_error=False) + except Exception as err: + return ToolExecutionResult(value=f"Execution error in {tool_call.name}: {_fmt_exc(err)}", is_error=True) + + @staticmethod + def _append_tool_turn( + history: History, + *, + inputs: dict[str, object] | None = None, + next_thought, + tool_calls: ToolCalls, + tool_results: list[ToolExecutionResult], + ) -> None: + history.append( + { + **(inputs or {}), + "next_thought": next_thought, + "tool_calls": tool_calls, + "tool_call_results": ToolCallResults.from_tool_calls_and_values( + tool_calls.tool_calls, + [tool_result.value for tool_result in tool_results], + [tool_result.is_error for tool_result in tool_results], + ), + } + ) + + def _render_history_as_text(self, history: History) -> str: + lines = [] + for event in history.messages: + thought = event.get("next_thought") + tool_calls = event.get("tool_calls") + tool_call_results = event.get("tool_call_results") + if thought: + lines.append(f"[Thought] {thought}") + if isinstance(tool_calls, ToolCalls): + for tool_call in tool_calls.tool_calls: + args = ", ".join(f"{key}={value!r}" for key, value in (tool_call.args or {}).items()) + lines.append(f"[Action] {tool_call.name}({args})") + if isinstance(tool_call_results, ToolCallResults): + for result in tool_call_results.tool_call_results: + prefix = "[Error]" if result.is_error else "[Observation]" + lines.append(f"{prefix} {result.name}: {result.value}") + for key, value in event.items(): + if key in {"next_thought", "tool_calls", "tool_call_results"}: + continue + label = "[Input]" if key in self.signature.input_fields else "[Output]" + lines.append(f"{label} {key}: {value}") + return "\n".join(lines) + + +def _fmt_exc(err: BaseException, *, limit: int = 5) -> str: + return "\n" + "".join(traceback.format_exception(type(err), err, err.__traceback__, limit=limit)).strip() diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.py index 754cbf54de..f5376ce5cd 100644 --- a/dspy/streaming/streaming_listener.py +++ b/dspy/streaming/streaming_listener.py @@ -145,13 +145,15 @@ def receive(self, chunk: ModelResponseStream): and issubclass(self._output_type, Type) and self._output_type.is_streamable() ): - if parsed_chunk := self._output_type.parse_stream_chunk(chunk): + parsed_chunk = self._output_type.parse_stream_chunk(chunk) + if parsed_chunk is not None: return StreamResponse( self.predict_name, self.signature_field_name, parsed_chunk, is_last_chunk=self.stream_end, ) + return None # For non-custom streamable types, the streaming chunks come from the content field of the ModelResponseStream. try: diff --git a/dspy/teleprompt/avatar_optimizer.py b/dspy/teleprompt/avatar_optimizer.py index ddba74e5f2..ea26270085 100644 --- a/dspy/teleprompt/avatar_optimizer.py +++ b/dspy/teleprompt/avatar_optimizer.py @@ -28,23 +28,18 @@ class Comparator(dspy.Signature): (3) Lastly, specify the modification in tools used that can lead to improved performance on the negative inputs.""" instruction: str = dspy.InputField( - prefix="Instruction: ", desc="Instruction for the actor to execute the task", ) actions: list[str] = dspy.InputField( - prefix="Actions: ", desc="Actions actor can take to complete the task", ) pos_input_with_metrics: list[EvalResult] = dspy.InputField( - prefix="Positive Inputs: ", desc="Positive inputs along with their score on a evaluation metric and actions taken", ) neg_input_with_metrics: list[EvalResult] = dspy.InputField( - prefix="Negative Inputs: ", desc="Negative inputs along with their score on a evaluation metric and actions taken", ) feedback: str = dspy.OutputField( - prefix="Feedback: ", desc="Feedback for the actor to improve the performance of negative inputs", ) @@ -59,15 +54,12 @@ class FeedbackBasedInstruction(dspy.Signature): Make sure that the new instruction talks about how to use the tools effectively and should be no more than 3 paragraphs long. The previous instruction contains general guidelines that you must retain in the new instruction.""" previous_instruction: str = dspy.InputField( - prefix="Previous Instruction: ", desc="Previous instruction for the actor to execute the task", ) feedback: str = dspy.InputField( - prefix="Feedback: ", desc="Feedback for the actor to improve the performance of negative inputs", ) new_instruction: str = dspy.OutputField( - prefix="New Instruction: ", desc="New instruction for the actor to execute the task", ) diff --git a/dspy/utils/exceptions.py b/dspy/utils/exceptions.py index 928bbe351b..dbfd79ae1c 100644 --- a/dspy/utils/exceptions.py +++ b/dspy/utils/exceptions.py @@ -1,25 +1,144 @@ +from __future__ import annotations from dspy.signatures.signature import Signature -class ContextWindowExceededError(Exception): - """Raised when the prompt exceeds the model's context window. +class DSPyError(Exception): + """Base class for DSPy errors with structured metadata.""" - Any `BaseLM` subclass should raise this error (or a subclass of it) when the - request fails because the input is too long for the model. Adapters and some - modules rely on catching this specific type to decide whether a fallback - retry is appropriate. + default_code: str | None = None - Args: - model: The model identifier that rejected the request. - message: Description of the error. Defaults to `"Context window exceeded"`. - """ - - def __init__(self, *, model: str | None = None, message: str = "Context window exceeded"): + def __init__( + self, + message: str = "", + *, + code: str | None = None, + model: str | None = None, + provider: str | None = None, + provider_code: str | None = None, + status: int | None = None, + request_id: str | None = None, + retry_after: float | None = None, + ): + self.message = message + self.code = code or self.default_code self.model = model - msg = message + self.provider = provider + self.provider_code = provider_code + self.status = status + self.request_id = request_id + self.retry_after = retry_after + prefix = f"[{model}] " if model else "" - super().__init__(f"{prefix}{msg}") + super().__init__(f"{prefix}{message}" if message else prefix.rstrip()) + + +class LMError(DSPyError): + """Base class for language model errors.""" + + default_code = "lm_error" + + +class LMTransportError(LMError): + """The LM request failed before the provider returned a response.""" + + default_code = "transport" + + +class LMConfigurationError(LMError): + """The LM or provider client is not configured correctly.""" + + default_code = "configuration" + + +class LMNotConfiguredError(LMConfigurationError): + """The LM is missing required provider configuration or credentials.""" + + default_code = "not_configured" + + +class LMUnsupportedFeatureError(LMError): + """The LM does not support a requested feature.""" + + default_code = "unsupported_feature" + + def __init__( + self, + message: str = "", + *, + features: list[str] | None = None, + issues: list[str] | None = None, + **kwargs, + ): + self.features = list(features or []) + self.issues = list(issues or []) + super().__init__(message, **kwargs) + + +class LMProviderError(LMError): + """The provider returned an error response.""" + + default_code = "provider" + + +class LMAuthError(LMProviderError): + """The provider rejected the request because authentication failed.""" + + default_code = "auth" + + +class LMBillingError(LMProviderError): + """The provider rejected the request because billing or quota failed.""" + + default_code = "billing" + + +class LMRateLimitError(LMProviderError): + """The provider rate-limited the request.""" + + default_code = "rate_limit" + + +class LMInvalidRequestError(LMProviderError): + """The provider rejected the request shape or resource.""" + + default_code = "invalid_request" + + +class ContextWindowExceededError(LMInvalidRequestError): + """Raised when the prompt exceeds the model's context window.""" + + default_code = "context_window_exceeded" + + def __init__( + self, + *, + model: str | None = None, + message: str = "Context window exceeded", + **kwargs, + ): + super().__init__(message, model=model, **kwargs) + + +class LMUnsupportedModelError(LMInvalidRequestError): + """The requested model is unavailable or unsupported by the provider.""" + + default_code = "unsupported_model" + + +class LMTimeoutError(LMProviderError): + """The provider request timed out.""" + + default_code = "timeout" + + +class LMServerError(LMProviderError): + """The provider failed while handling the request.""" + + default_code = "server" + + +RETRYABLE_LM_ERRORS = (LMRateLimitError, LMTimeoutError, LMServerError, LMTransportError) class AdapterParseError(Exception): diff --git a/tests/adapters/conftest.py b/tests/adapters/conftest.py new file mode 100644 index 0000000000..e1d089dcee --- /dev/null +++ b/tests/adapters/conftest.py @@ -0,0 +1,55 @@ +import dspy + + +class StopAdapterCallCapture(BaseException): + """Stop adapter execution after capturing the LM call. + + The exact-format tests assert the adapter-to-LM boundary: the messages and + keyword arguments passed to the LM. Raising here avoids needing to craft a + parseable LM response for every signature under test. + """ + + +class CapturingLM(dspy.BaseLM): + def __init__(self, source_lm=None): + source_lm = source_lm or dspy.utils.DummyLM([{}]) + super().__init__( + model=source_lm.model, + model_type=source_lm.model_type, + cache=source_lm.cache, + **source_lm.kwargs, + ) + self.source_lm = source_lm + self.calls = [] + + @property + def supports_function_calling(self): + return self.source_lm.supports_function_calling + + @property + def supports_reasoning(self): + return self.source_lm.supports_reasoning + + @property + def supports_response_schema(self): + return self.source_lm.supports_response_schema + + @property + def supported_params(self): + return self.source_lm.supported_params + + def __call__(self, messages=None, **kwargs): + self.calls.append({"messages": messages, "kwargs": kwargs}) + raise StopAdapterCallCapture + + +def format_messages_and_lm_kwargs(adapter, signature, demos, inputs, lm_kwargs=None, lm=None): + capturing_lm = CapturingLM(lm) + try: + adapter(capturing_lm, dict(lm_kwargs or {}), signature, demos, inputs) + except StopAdapterCallCapture: + pass + + assert len(capturing_lm.calls) == 1 + call = capturing_lm.calls[0] + return call["messages"], call["kwargs"] diff --git a/tests/adapters/test_adapter_base.py b/tests/adapters/test_adapter_base.py new file mode 100644 index 0000000000..a7cbf920a7 --- /dev/null +++ b/tests/adapters/test_adapter_base.py @@ -0,0 +1,63 @@ +import dspy +from dspy.core.types import LMConfig, LMToolChoice +from tests.adapters.conftest import format_messages_and_lm_kwargs + + +def test_adapter_does_not_require_native_tool_call_format_kwarg_when_unused(): + class QA(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + class OldStyleFormatAdapter(dspy.Adapter): + def format(self, signature, demos, inputs): + return [{"role": "user", "content": inputs["question"]}] + + def parse(self, signature, completion): + return {"answer": completion} + + messages, lm_kwargs = format_messages_and_lm_kwargs( + OldStyleFormatAdapter(), + QA, + [], + {"question": "Q"}, + ) + + assert messages == [{"role": "user", "content": "Q"}] + assert lm_kwargs == {} + + +def test_call_context_does_not_carry_mutable_extra_state(): + context = dspy.Adapter(allow_parallel_tool_calls=False).build_call_context( + dspy.utils.DummyLM([{}]), + {"temperature": 0.2}, + ) + + assert not hasattr(context, "extra") + assert context.allow_parallel_tool_calls is False + assert context.lm_kwargs == {"temperature": 0.2} + assert context.lm_default_kwargs["temperature"] == 0.0 + + +def test_adapter_merges_lm_config_patches_without_losing_nested_fields(): + adapter = dspy.Adapter() + left = LMConfig.from_kwargs(reasoning={"effort": "low"}, cache=True, custom="left") + right = LMConfig.from_kwargs(reasoning={"max_tokens": 64}, rollout_id="rollout", other="right") + + merged = adapter._merge_lm_config(left, right) + + assert merged.reasoning.effort == "low" + assert merged.reasoning.max_tokens == 64 + assert merged.cache.enabled is True + assert merged.cache.rollout_id == "rollout" + assert merged.extensions == {"custom": "left", "other": "right"} + + +def test_adapter_lm_config_patch_preserves_explicit_tool_choice_mode(): + adapter = dspy.Adapter() + merged_kwargs = adapter._merge_config_kwargs( + {"tool_choice": "none"}, + LMConfig(tool_choice=LMToolChoice(parallel=False)), + ) + + assert merged_kwargs["tool_choice"]["mode"] == "none" + assert merged_kwargs["tool_choice"]["parallel"] is False diff --git a/tests/adapters/test_adapter_field_parts.py b/tests/adapters/test_adapter_field_parts.py new file mode 100644 index 0000000000..01a625de39 --- /dev/null +++ b/tests/adapters/test_adapter_field_parts.py @@ -0,0 +1,67 @@ +import dspy +from dspy.core.types import LMOutput, LMTextPart + + +def test_base_value_helpers_round_trip_text_field_parts(): + class SimpleSignature(dspy.Signature): + answer: int = dspy.OutputField() + + adapter = dspy.ChatAdapter() + field_info = SimpleSignature.output_fields["answer"] + + parts = adapter.value_to_lm_parts(42, field_info) + + assert parts == [LMTextPart(text="42")] + assert adapter.lm_parts_to_value(parts, field_info) == 42 + + +def test_chat_adapter_wraps_and_slices_field_parts(): + class SimpleSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.ChatAdapter() + + wrapped = adapter.wrap_input_field_parts("question", [LMTextPart(text="What?")]) + assert wrapped == [LMTextPart(text="[[ ## question ## ]]\n"), LMTextPart(text="What?")] + + output = LMOutput(parts=[LMTextPart(text="[[ ## answer ## ]]\nParis\n\n[[ ## completed ## ]]")]) + fields = adapter.parse_output_fields_to_parts(output, SimpleSignature) + + assert fields == {"answer": [LMTextPart(text="Paris")]} + + +def test_json_adapter_slices_output_fields_to_text_parts(): + class SimpleSignature(dspy.Signature): + answer: str = dspy.OutputField() + score: float = dspy.OutputField() + + adapter = dspy.JSONAdapter() + output = LMOutput(parts=[LMTextPart(text='{"answer": "Paris", "score": 0.9}')]) + + fields = adapter.parse_output_fields_to_parts(output, SimpleSignature) + + assert fields == { + "answer": [LMTextPart(text="Paris")], + "score": [LMTextPart(text="0.9")], + } + + +def test_xml_adapter_wraps_and_slices_field_parts(): + class SimpleSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.XMLAdapter() + + wrapped = adapter.wrap_output_field_parts("answer", [LMTextPart(text="Paris")]) + assert wrapped == [ + LMTextPart(text="\n"), + LMTextPart(text="Paris"), + LMTextPart(text="\n"), + ] + + output = LMOutput(parts=[LMTextPart(text="Paris")]) + fields = adapter.parse_output_fields_to_parts(output, SimpleSignature) + + assert fields == {"answer": [LMTextPart(text="Paris")]} diff --git a/tests/adapters/test_adapter_migration_robustness.py b/tests/adapters/test_adapter_migration_robustness.py new file mode 100644 index 0000000000..a391defd07 --- /dev/null +++ b/tests/adapters/test_adapter_migration_robustness.py @@ -0,0 +1,83 @@ +import dspy +from dspy.utils.exceptions import AdapterParseError + + +def add(a: int, b: int) -> int: + return a + b + + +class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + def __init__(self, output): + self.output = output + self.messages = None + self.kwargs = None + + def __call__(self, messages, **kwargs): + self.messages = messages + self.kwargs = kwargs + return [self.output] + + +def test_native_tool_response_can_combine_visible_text_and_tool_calls(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + answer: str = dspy.OutputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + lm = NativeToolLM( + { + "text": "[[ ## answer ## ]]\nworking\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "id": "call_add", + "type": "function", + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + } + ], + } + ) + + result = dspy.ChatAdapter(use_native_function_calling=True)( + lm, + {}, + ToolSignature, + [], + {"question": "What is 1+2?", "tools": [dspy.Tool(add)]}, + )[0] + + assert result["answer"] == "working" + assert result["tool_calls"].tool_calls[0].id == "call_add" + assert result["tool_calls"].tool_calls[0].args == {"a": 1, "b": 2} + + +def test_tool_type_handler_does_not_override_explicit_tool_choice_mode(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + lm = NativeToolLM({"text": None}) + adapter = dspy.ChatAdapter(use_native_function_calling=True, allow_parallel_tool_calls=False) + + try: + adapter( + lm, + {"tool_choice": "none"}, + ToolSignature, + [], + {"question": "What is 1+2?", "tools": [dspy.Tool(add)]}, + ) + except AdapterParseError: + pass + + assert lm.kwargs["tool_choice"] == "none" + assert lm.kwargs["parallel_tool_calls"] is False + assert lm.kwargs["tools"][0]["function"]["parameters"]["properties"]["a"]["type"] == "integer" diff --git a/tests/adapters/test_baml_adapter.py b/tests/adapters/test_baml_adapter.py index 5cc1481334..e8eaa9f6ee 100644 --- a/tests/adapters/test_baml_adapter.py +++ b/tests/adapters/test_baml_adapter.py @@ -8,6 +8,7 @@ import dspy from dspy.adapters.baml_adapter import COMMENT_SYMBOL, INDENTATION, BAMLAdapter +from tests.adapters.conftest import format_messages_and_lm_kwargs # Test fixtures - Pydantic models for testing @@ -51,6 +52,84 @@ class CircularModel(pydantic.BaseModel): field: "CircularModel" +def test_baml_adapter_format_exact_messages_for_simple_signature_with_demo(): + class QA(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(BAMLAdapter(), QA, [{"question": "Q1", "answer": "A1"}], {"question": "Q2"}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "Output field `answer` should be of type: string\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`."}, + {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, + {"role": "assistant", "content": '{\n "answer": "A1"\n}'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "Respond with a JSON object in the following order of fields: `answer`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_baml_adapter_format_exact_messages_with_nested_output(): + class BamlNested(pydantic.BaseModel): + value: int + tags: list[str] + + class TypedSignature(dspy.Signature): + question: str = dspy.InputField() + answer: BamlNested = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(BAMLAdapter(), TypedSignature, [], {"question": "Q"}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (BamlNested):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "Output field `answer` should be of type: {\n" + " value: int,\n" + " tags: string[],\n" + "}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q\n" + "\n" + "Respond with a JSON object in the following order of fields: `answer` (must be " + "formatted as a valid Python BamlNested)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + def test_baml_adapter_basic_schema_generation(): """Test that BAMLAdapter generates simplified schemas for Pydantic models.""" diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py index 86c662b3ba..bdc2b73432 100644 --- a/tests/adapters/test_chat_adapter.py +++ b/tests/adapters/test_chat_adapter.py @@ -1,3 +1,4 @@ +import re from typing import Literal from unittest import mock @@ -6,7 +7,8 @@ from litellm.utils import ChatCompletionMessageToolCall, Choices, Function, Message, ModelResponse import dspy -from dspy.experimental import Citations +from dspy.experimental import Citations, Document +from tests.adapters.conftest import format_messages_and_lm_kwargs @pytest.mark.parametrize( @@ -99,6 +101,1340 @@ async def test_chat_adapter_async_call(): assert result == [{"answer": "Paris"}] +def test_chat_adapter_format_exact_messages_for_simple_signature(): + class QA(dspy.Signature): + """Answer the question.""" + + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), QA, [], {"question": "What is the capital of France?"}) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str): +All interactions will be structured in the following way, with the appropriate values filled in. + +[[ ## question ## ]] +{question} + +[[ ## answer ## ]] +{answer} + +[[ ## completed ## ]] +In adhering to this structure, your objective is:\x20 + Answer the question.""", + }, + { + "role": "user", + "content": """[[ ## question ## ]] +What is the capital of France? + +Respond with the corresponding output fields, starting with the field `[[ ## answer ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`.""", + }, + ] + + +def test_chat_adapter_format_exact_messages_with_demo_and_typed_outputs(): + class MultiAnswer(dspy.Signature): + """Answer the question with multiple answers and scores""" + + question: str = dspy.InputField() + answers: list[str] = dspy.OutputField() + scores: list[float] = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + MultiAnswer, + demos=[{"question": "Q1", "answers": ["A1", "A2"], "scores": [0.1, 0.9]}], + inputs={"question": "Q2"}, + ) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answers` (list[str]):\x20 +2. `scores` (list[float]): +All interactions will be structured in the following way, with the appropriate values filled in. + +[[ ## question ## ]] +{question} + +[[ ## answers ## ]] +{answers} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "string"}} + +[[ ## scores ## ]] +{scores} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "number"}} + +[[ ## completed ## ]] +In adhering to this structure, your objective is:\x20 + Answer the question with multiple answers and scores""", + }, + {"role": "user", "content": """[[ ## question ## ]] +Q1"""}, + { + "role": "assistant", + "content": """[[ ## answers ## ]] +["A1", "A2"] + +[[ ## scores ## ]] +[0.1, 0.9] + +[[ ## completed ## ]] +""", + }, + { + "role": "user", + "content": """[[ ## question ## ]] +Q2 + +Respond with the corresponding output fields, starting with the field `[[ ## answers ## ]]` (must be formatted as a valid Python list[str]), then `[[ ## scores ## ]]` (must be formatted as a valid Python list[float]), and then ending with the marker for `[[ ## completed ## ]]`.""", + }, + ] + + +def test_chat_adapter_format_exact_messages_with_nested_pydantic_models(): + class Address(pydantic.BaseModel): + city: str + country: str + + class Person(pydantic.BaseModel): + name: str + address: Address + tags: list[str] + + class Summary(pydantic.BaseModel): + headline: str + score: float + + class PydanticSignature(dspy.Signature): + person: Person = dspy.InputField() + summary: Summary = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + PydanticSignature, + [], + {"person": Person(name="Ada", address=Address(city="London", country="UK"), tags=["math", "code"])}, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `person` (Person):\n' + 'Your output fields are:\n' + '1. `summary` (Summary):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '[[ ## person ## ]]\n' + '{person}\n' + '\n' + '[[ ## summary ## ]]\n' + '{summary} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "properties": {"headline": {"type": "string", "title": ' + '"Headline"}, "score": {"type": "number", "title": "Score"}}, "required": ' + '["headline", "score"], "title": "Summary"}\n' + '\n' + '[[ ## completed ## ]]\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `person`, produce the fields `summary`.'}, + {"role": "user", + "content": '[[ ## person ## ]]\n' + '{"name": "Ada", "address": {"city": "London", "country": "UK"}, "tags": ["math", ' + '"code"]}\n' + '\n' + 'Respond with the corresponding output fields, starting with the field `[[ ## summary ' + '## ]]` (must be formatted as a valid Python Summary), and then ending with the ' + 'marker for `[[ ## completed ## ]]`.'}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_incomplete_demo(): + class IncompleteDemoSignature(dspy.Signature): + question: str = dspy.InputField() + context: str = dspy.InputField() + answer: str = dspy.OutputField() + confidence: float = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + IncompleteDemoSignature, + [{"question": "Q1", "answer": "A1"}], + {"question": "Q2", "context": "C2"}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str): \n" + "2. `context` (str):\n" + "Your output fields are:\n" + "1. `answer` (str): \n" + "2. `confidence` (float):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## context ## ]]\n" + "{context}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## confidence ## ]]\n" + "{confidence} # note: the value you produce must be a single float value\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `context`, produce the fields `answer`, " + "`confidence`."}, + {"role": "user", + "content": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## question ## ]]\n" + "Q1"}, + {"role": "assistant", + "content": "[[ ## answer ## ]]\n" + "A1\n" + "\n" + "[[ ## confidence ## ]]\n" + "Not supplied for this particular example.\n" + "\n" + "[[ ## completed ## ]]\n"}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "[[ ## context ## ]]\n" + "C2\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, then `[[ ## confidence ## ]]` (must be formatted as a valid Python float), " + "and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_history(): + class HistorySignature(dspy.Signature): + history: dspy.History = dspy.InputField() + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + history = dspy.History( + messages=[ + {"question": "What is 1+1?", "answer": "2"}, + {"question": "What is 2+2?", "answer": "4"}, + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + HistorySignature, + [], + {"history": history, "question": "What is 3+3?"}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`."}, + {"role": "user", "content": "[[ ## question ## ]]\nWhat is 1+1?"}, + {"role": "assistant", "content": "[[ ## answer ## ]]\n2\n\n[[ ## completed ## ]]\n"}, + {"role": "user", "content": "[[ ## question ## ]]\nWhat is 2+2?"}, + {"role": "assistant", "content": "[[ ## answer ## ]]\n4\n\n[[ ## completed ## ]]\n"}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "What is 3+3?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_list_value_for_string_input(): + class ListAsStringSignature(dspy.Signature): + context: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), ListAsStringSignature, [], {"context": ["alpha", "beta"]}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `context` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## context ## ]]\n" + "{context}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `context`, produce the fields `answer`."}, + {"role": "user", + "content": "[[ ## context ## ]]\n" + "[1] «alpha»\n" + "[2] «beta»\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_literal_output(): + class LiteralSignature(dspy.Signature): + question: str = dspy.InputField() + verdict: Literal["yes", "no"] = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), LiteralSignature, [], {"question": "Is the sky blue?"}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `verdict` (Literal['yes', 'no']):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## verdict ## ]]\n" + "{verdict} # note: the value you produce must exactly match (no extra " + "characters) one of: yes; no\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `verdict`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Is the sky blue?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## verdict " + "## ]]` (must be formatted as a valid Python Literal['yes', 'no']), and then ending " + "with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_multimodal_custom_type_inputs(): + class CustomTypeSignature(dspy.Signature): + image: dspy.Image = dspy.InputField() + audio: dspy.Audio = dspy.InputField() + file: dspy.File = dspy.InputField() + document: Document = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + CustomTypeSignature, + [], + { + "image": dspy.Image("https://example.com/cat.png"), + "audio": dspy.Audio(data="QUJD", audio_format="wav"), + "file": dspy.File.from_file_id("file-123", filename="notes.txt"), + "document": Document(data="Alpha beta", title="Doc"), + }, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `image` (Image): \n" + "2. `audio` (Audio): \n" + "3. `file` (File): \n" + "4. `document` (Document): \n" + " Type description of Document: A document containing text content that can be " + "referenced and cited. Include the full text content and optionally a title for " + "proper referencing.\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## image ## ]]\n" + "{image}\n" + "\n" + "[[ ## audio ## ]]\n" + "{audio}\n" + "\n" + "[[ ## file ## ]]\n" + "{file}\n" + "\n" + "[[ ## document ## ]]\n" + "{document}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `image`, `audio`, `file`, `document`, produce the fields " + "`answer`."}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "text", "text": "\n\n[[ ## audio ## ]]\n"}, + {"type": "input_audio", "input_audio": {"data": "QUJD", "format": "wav"}}, + {"type": "text", "text": "\n\n[[ ## file ## ]]\n"}, + {"type": "file", "file": {"file_id": "file-123", "filename": "notes.txt"}}, + {"type": "text", "text": "\n\n[[ ## document ## ]]\n"}, + {"type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": "Alpha beta"}, + "citations": {"enabled": True}, + "title": "Doc"}, + {"type": "text", + "text": "\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ " + "## answer ## ]]`, and then ending with the marker for `[[ ## completed ## " + "]]`."}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_with_history_demo_pydantic_tools_and_image(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class Location(pydantic.BaseModel): + city: str + country: str + + class Profile(pydantic.BaseModel): + name: str + location: Location + interests: list[str] + + class AnswerCard(pydantic.BaseModel): + answer: str + sources: list[str] + + class RichRenderingSignature(dspy.Signature): + """Answer using all supplied context.""" + + history: dspy.History = dspy.InputField() + image: dspy.Image = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + profile: Profile = dspy.InputField() + question: str = dspy.InputField() + answer: AnswerCard = dspy.OutputField() + + tool = dspy.Tool(search) + demo_profile = Profile( + name="Ada", + location=Location(city="London", country="UK"), + interests=["math", "machines"], + ) + current_profile = Profile( + name="Grace", + location=Location(city="Arlington", country="USA"), + interests=["compilers", "navy"], + ) + history = dspy.History( + messages=[ + { + "profile": demo_profile, + "question": "Who is Ada?", + "answer": AnswerCard(answer="Ada is a mathematician.", sources=["memory"]), + } + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + RichRenderingSignature, + demos=[ + { + "image": dspy.Image("https://example.com/demo.png"), + "tools": [tool], + "profile": demo_profile, + "question": "What should we mention?", + "answer": AnswerCard(answer="Mention analytical engines.", sources=["demo"]), + } + ], + inputs={ + "history": history, + "image": dspy.Image("https://example.com/current.png"), + "tools": [tool], + "profile": current_profile, + "question": "What should the answer include?", + }, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `image` (Image): \n' + '2. `tools` (list[Tool]): \n' + '3. `profile` (Profile): \n' + '4. `question` (str):\n' + 'Your output fields are:\n' + '1. `answer` (AnswerCard):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '[[ ## image ## ]]\n' + '{image}\n' + '\n' + '[[ ## tools ## ]]\n' + '{tools}\n' + '\n' + '[[ ## profile ## ]]\n' + '{profile}\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + '[[ ## answer ## ]]\n' + '{answer} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "properties": {"answer": {"type": "string", "title": "Answer"}, ' + '"sources": {"type": "array", "items": {"type": "string"}, "title": "Sources"}}, ' + '"required": ["answer", "sources"], "title": "AnswerCard"}\n' + '\n' + '[[ ## completed ## ]]\n' + 'In adhering to this structure, your objective is: \n' + ' Answer using all supplied context.'}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' + '"interests": ["math", "machines"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'What should we mention?'}]}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Mention analytical engines.", "sources": ["demo"]}\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, "interests": ' + '["math", "machines"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'Who is Ada?'}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Ada is a mathematician.", "sources": ["memory"]}\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": [{"type": "text", + "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' + '"interests": ["compilers", "navy"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'What should the answer include?\n' + '\n' + 'Respond with the corresponding output fields, starting with the field `[[ ' + '## answer ## ]]` (must be formatted as a valid Python AnswerCard), and ' + 'then ending with the marker for `[[ ## completed ## ]]`.'}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_base_custom_type_input(): + class Event(dspy.Type): + label: str + + def format(self): + return [{"type": "event", "event": {"label": self.label}}] + + @classmethod + def description(cls): + return "An event block." + + class EventSignature(dspy.Signature): + event: Event = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), EventSignature, [], {"event": Event(label="launch")}) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `event` (Event): \n" + " Type description of Event: An event block.\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## event ## ]]\n" + "{event}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `event`, produce the fields `answer`."}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## event ## ]]\n"}, + {"type": "event", "event": {"label": "launch"}}, + {"type": "text", + "text": "\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ " + "## answer ## ]]`, and then ending with the marker for `[[ ## completed ## " + "]]`."}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_citations_output_demo(): + class CitationSignature(dspy.Signature): + question: str = dspy.InputField() + citations: Citations = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + CitationSignature, + [ + { + "question": "Q1", + "citations": Citations.from_dict_list( + [ + { + "cited_text": "alpha", + "document_index": 0, + "start_char_index": 0, + "end_char_index": 5, + } + ] + ), + } + ], + {"question": "Q2"}, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `citations` (Citations): \n' + ' Type description of Citations: Citations with quoted text and source references. ' + 'Include the exact text being cited and information about its source.\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + '[[ ## citations ## ]]\n' + '{citations} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "$defs": {"Citation": {"type": "object", "description": ' + '"Individual citation with character location information.", "properties": {"type": ' + '{"type": "string", "default": "char_location", "title": "Type"}, "cited_text": ' + '{"type": "string", "title": "Cited Text"}, "document_index": {"type": "integer", ' + '"title": "Document Index"}, "document_title": {"anyOf": [{"type": "string"}, ' + '{"type": "null"}], "default": null, "title": "Document Title"}, "end_char_index": ' + '{"type": "integer", "title": "End Char Index"}, "start_char_index": {"type": ' + '"integer", "title": "Start Char Index"}, "supported_text": {"anyOf": [{"type": ' + '"string"}, {"type": "null"}], "default": null, "title": "Supported Text"}}, ' + '"required": ["cited_text", "document_index", "start_char_index", "end_char_index"], ' + '"title": "Citation"}}, "description": "Experimental: This class may change or be ' + 'removed in a future release without warning (introduced in v3.0.4).\\n\\nCitations ' + 'extracted from an LM response with source references.\\n\\n This type represents ' + 'citations returned by language models that support\\n citation extraction, ' + "particularly Anthropic's Citations API through LiteLLM.\\n Citations include the " + 'quoted text and source information.\\n\\n Examples:\\n ```python\\n ' + 'import os\\n import dspy\\n from dspy.signatures import ' + 'Signature\\n from dspy.experimental import Citations, Document\\n ' + 'os.environ[\\"ANTHROPIC_API_KEY\\"] = \\"YOUR_ANTHROPIC_API_KEY\\"\\n\\n ' + "class AnswerWithSources(Signature):\\n '''Answer questions using provided " + "documents with citations.'''\\n documents: list[Document] = " + 'dspy.InputField()\\n question: str = dspy.InputField()\\n ' + 'answer: str = dspy.OutputField()\\n citations: Citations = ' + 'dspy.OutputField()\\n\\n # Create documents to provide as sources\\n ' + 'docs = [\\n Document(\\n data=\\"The Earth orbits the Sun ' + 'in an elliptical path.\\",\\n title=\\"Basic Astronomy ' + 'Facts\\"\\n ),\\n Document(\\n data=\\"Water ' + 'boils at 100°C at standard atmospheric pressure.\\",\\n ' + 'title=\\"Physics Fundamentals\\",\\n metadata={\\"author\\": \\"Dr. ' + 'Smith\\", \\"year\\": 2023}\\n )\\n ]\\n\\n # Use with a ' + 'model that supports citations like Claude\\n lm = ' + 'dspy.LM(\\"anthropic/claude-opus-4-1-20250805\\")\\n predictor = ' + 'dspy.Predict(AnswerWithSources)\\n result = predictor(documents=docs, ' + 'question=\\"What temperature does water boil?\\", lm=lm)\\n\\n for citation ' + 'in result.citations.citations:\\n print(citation.format())\\n ' + '```\\n ", "properties": {"citations": {"type": "array", "items": {"$ref": ' + '"#/$defs/Citation"}, "title": "Citations"}}, "required": ["citations"], "title": ' + '"Citations"}\n' + '\n' + '[[ ## completed ## ]]\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `citations`.'}, + {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, + {"role": "assistant", + "content": '[[ ## citations ## ]]\n' + '<>[{"type": "char_location", "cited_text": "alpha", ' + '"document_index": 0, "start_char_index": 0, "end_char_index": ' + '5}]<>\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## " + "citations ## ]]` (must be formatted as a valid Python Citations), and then ending " + "with the marker for `[[ ## completed ## ]]`."}] + def normalize_citations_schema_description(content): + return re.sub( + r'"description": ".*?", "properties":', + '"description": "", "properties":', + content, + ) + + messages[0]["content"] = normalize_citations_schema_description(messages[0]["content"]) + expected_messages[0]["content"] = normalize_citations_schema_description(expected_messages[0]["content"]) + + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_and_lm_kwargs_with_native_citations(): + class AnthropicLM(dspy.utils.DummyLM): + def __init__(self): + super().__init__([{}]) + self.model = "anthropic/claude-3-5-sonnet" + + class CitationSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + citations: Citations = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(), + CitationSignature, + [], + {"question": "Q?"}, + lm=AnthropicLM(), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`, `citations`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_preserves_passthrough_lm_kwargs(): + class PassthroughSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(), + PassthroughSignature, + [], + {"question": "Q?"}, + lm_kwargs={"temperature": 0.7, "max_tokens": 42, "stream": True, "cache": False}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `answer`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {"temperature": 0.7, "max_tokens": 42, "stream": True, "cache": False} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_and_lm_kwargs_with_native_reasoning(): + class ReasoningLM(dspy.utils.DummyLM): + @property + def supports_reasoning(self): + return True + + class NativeReasoningSignature(dspy.Signature): + question: str = dspy.InputField() + reasoning: dspy.Reasoning = dspy.OutputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(), + NativeReasoningSignature, + [], + {"question": "Q?"}, + lm=ReasoningLM([{}]), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `reasoning`, `answer`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## answer " + "## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {"reasoning_effort": "low"} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_reasoning_and_code_outputs(): + python_code = dspy.Code["python"] + + class CodeSignature(dspy.Signature): + question: str = dspy.InputField() + reasoning: dspy.Reasoning = dspy.OutputField() + code: python_code = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + CodeSignature, + [{"question": "Q1", "reasoning": dspy.Reasoning(content="Think"), "code": python_code(code="print('hi')")}], + {"question": "Q2"}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "Your output fields are:\n" + "1. `reasoning` (str): \n" + "2. `code` (Code_python): \n" + " Type description of Code_python: Code represented in a string, specified in the " + "`code` field. If this is an output field, the code field should follow the markdown " + "code block format, e.g. \n" + "```python\n" + "{code}\n" + "```\n" + "Programming language: python\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## reasoning ## ]]\n" + "{reasoning}\n" + "\n" + "[[ ## code ## ]]\n" + "{code}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, produce the fields `reasoning`, `code`."}, + {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, + {"role": "assistant", + "content": "[[ ## reasoning ## ]]\n" + "Think\n" + "\n" + "[[ ## code ## ]]\n" + "print('hi')\n" + "\n" + "[[ ## completed ## ]]\n"}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "Respond with the corresponding output fields, starting with the field `[[ ## " + "reasoning ## ]]` (must be formatted as a valid Python str), then `[[ ## code ## ]]` " + "(must be formatted as a valid Python Code_python), and then ending with the marker " + "for `[[ ## completed ## ]]`."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_and_lm_kwargs_with_native_tool_calling(): + class FunctionCallingLM(dspy.utils.DummyLM): + @property + def supports_function_calling(self): + return True + + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class NativeToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(use_native_function_calling=True), + NativeToolSignature, + [], + {"question": "Q?", "tools": [dspy.Tool(search)]}, + lm=FunctionCallingLM([{}]), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?"}] + assert messages == expected_messages + expected_lm_kwargs = {"tools": [{"type": "function", + "function": {"name": "search", + "description": "Search for documents.", + "parameters": {"type": "object", + "properties": {"query": {"type": "string"}, + "k": {"type": "integer", "default": 3}}, + "required": ["query", "k"]}}}]} + assert lm_kwargs == expected_lm_kwargs + +def test_chat_adapter_format_exact_messages_with_tool_input(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + ToolSignature, + [], + {"question": "Q?", "tools": [dspy.Tool(search)]}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str): \n" + "2. `tools` (list[Tool]):\n" + "Your output fields are:\n" + "1. `answer` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "\n" + "[[ ## tools ## ]]\n" + "{tools}\n" + "\n" + "[[ ## answer ## ]]\n" + "{answer}\n" + "\n" + "[[ ## completed ## ]]\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `tools`, produce the fields `answer`."}, + {"role": "user", + "content": '[[ ## question ## ]]\n' + 'Q?\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It takes ' + "arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', 'default': " + '3}}."]\n' + '\n' + 'Respond with the corresponding output fields, starting with the field `[[ ## answer ' + '## ]]`, and then ending with the marker for `[[ ## completed ## ]]`.'}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_chat_adapter_format_exact_messages_kitchen_sink(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class Event(dspy.Type): + label: str + + def format(self): + return [{"type": "event", "event": {"label": self.label}}] + + @classmethod + def description(cls): + return "An event block." + + class Location(pydantic.BaseModel): + city: str + country: str + + class Profile(pydantic.BaseModel): + name: str + location: Location + interests: list[str] + + class AnswerCard(pydantic.BaseModel): + answer: str + sources: list[str] + + class KitchenSinkSignature(dspy.Signature): + """Answer carefully using every available signal.""" + + history: dspy.History = dspy.InputField() + image: dspy.Image = dspy.InputField() + audio: dspy.Audio = dspy.InputField() + file: dspy.File = dspy.InputField() + document: Document = dspy.InputField() + event: Event = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + profile: Profile = dspy.InputField() + context: str = dspy.InputField() + question: str = dspy.InputField() + answer: AnswerCard = dspy.OutputField() + verdict: Literal["yes", "no"] = dspy.OutputField() + confidence: float = dspy.OutputField() + + tool = dspy.Tool(search) + demo_profile = Profile( + name="Ada", + location=Location(city="London", country="UK"), + interests=["math", "machines"], + ) + current_profile = Profile( + name="Grace", + location=Location(city="Arlington", country="USA"), + interests=["compilers", "navy"], + ) + history = dspy.History( + messages=[ + { + "profile": demo_profile, + "context": ["old note", "older note"], + "question": "Who is Ada?", + "answer": AnswerCard(answer="Ada is a mathematician.", sources=["memory"]), + "verdict": "yes", + "confidence": 0.8, + } + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.ChatAdapter(), + KitchenSinkSignature, + demos=[ + { + "image": dspy.Image("https://example.com/demo.png"), + "audio": dspy.Audio(data="REVNTw==", audio_format="wav"), + "file": dspy.File.from_file_id("file-demo", filename="demo.txt"), + "document": Document(data="Demo document", title="Demo Doc"), + "event": Event(label="demo-event"), + "tools": [tool], + "profile": demo_profile, + "context": ["demo context one", "demo context two"], + "question": "What should we mention?", + "answer": AnswerCard(answer="Mention analytical engines.", sources=["demo"]), + "verdict": "yes", + "confidence": 0.9, + }, + { + "question": "Incomplete example question", + "answer": AnswerCard(answer="Partial answer.", sources=["partial"]), + }, + ], + inputs={ + "history": history, + "image": dspy.Image("https://example.com/current.png"), + "audio": dspy.Audio(data="Q1VSUkVOVA==", audio_format="wav"), + "file": dspy.File.from_file_id("file-current", filename="current.txt"), + "document": Document(data="Current document", title="Current Doc"), + "event": Event(label="current-event"), + "tools": [tool], + "profile": current_profile, + "context": ["current context one", "current context two"], + "question": "What should the answer include?", + }, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `image` (Image): \n' + '2. `audio` (Audio): \n' + '3. `file` (File): \n' + '4. `document` (Document): \n' + ' Type description of Document: A document containing text content that can be ' + 'referenced and cited. Include the full text content and optionally a title for ' + 'proper referencing.\n' + '5. `event` (Event): \n' + ' Type description of Event: An event block.\n' + '6. `tools` (list[Tool]): \n' + '7. `profile` (Profile): \n' + '8. `context` (str): \n' + '9. `question` (str):\n' + 'Your output fields are:\n' + '1. `answer` (AnswerCard): \n' + "2. `verdict` (Literal['yes', 'no']): \n" + '3. `confidence` (float):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '[[ ## image ## ]]\n' + '{image}\n' + '\n' + '[[ ## audio ## ]]\n' + '{audio}\n' + '\n' + '[[ ## file ## ]]\n' + '{file}\n' + '\n' + '[[ ## document ## ]]\n' + '{document}\n' + '\n' + '[[ ## event ## ]]\n' + '{event}\n' + '\n' + '[[ ## tools ## ]]\n' + '{tools}\n' + '\n' + '[[ ## profile ## ]]\n' + '{profile}\n' + '\n' + '[[ ## context ## ]]\n' + '{context}\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + '[[ ## answer ## ]]\n' + '{answer} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "properties": {"answer": {"type": "string", "title": "Answer"}, ' + '"sources": {"type": "array", "items": {"type": "string"}, "title": "Sources"}}, ' + '"required": ["answer", "sources"], "title": "AnswerCard"}\n' + '\n' + '[[ ## verdict ## ]]\n' + '{verdict} # note: the value you produce must exactly match (no extra ' + 'characters) one of: yes; no\n' + '\n' + '[[ ## confidence ## ]]\n' + '{confidence} # note: the value you produce must be a single float value\n' + '\n' + '[[ ## completed ## ]]\n' + 'In adhering to this structure, your objective is: \n' + ' Answer carefully using every available signal.'}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, + {"type": "text", "text": "\n\n[[ ## audio ## ]]\n"}, + {"type": "input_audio", "input_audio": {"data": "REVNTw==", "format": "wav"}}, + {"type": "text", "text": "\n\n[[ ## file ## ]]\n"}, + {"type": "file", "file": {"file_id": "file-demo", "filename": "demo.txt"}}, + {"type": "text", "text": "\n\n[[ ## document ## ]]\n"}, + {"type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": "Demo document"}, + "citations": {"enabled": True}, + "title": "Demo Doc"}, + {"type": "text", "text": "\n\n[[ ## event ## ]]\n"}, + {"type": "event", "event": {"label": "demo-event"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' + '"interests": ["math", "machines"]}\n' + '\n' + '[[ ## context ## ]]\n' + '[1] «demo context one»\n' + '[2] «demo context two»\n' + '\n' + '[[ ## question ## ]]\n' + 'What should we mention?'}]}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Mention analytical engines.", "sources": ["demo"]}\n' + '\n' + '[[ ## verdict ## ]]\n' + 'yes\n' + '\n' + '[[ ## confidence ## ]]\n' + '0.9\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## question ## ]]\n" + "Incomplete example question"}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Partial answer.", "sources": ["partial"]}\n' + '\n' + '[[ ## verdict ## ]]\n' + 'Not supplied for this particular example. \n' + '\n' + '[[ ## confidence ## ]]\n' + 'Not supplied for this particular example.\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, "interests": ' + '["math", "machines"]}\n' + '\n' + '[[ ## context ## ]]\n' + '[1] «old note»\n' + '[2] «older note»\n' + '\n' + '[[ ## question ## ]]\n' + 'Who is Ada?'}, + {"role": "assistant", + "content": '[[ ## answer ## ]]\n' + '{"answer": "Ada is a mathematician.", "sources": ["memory"]}\n' + '\n' + '[[ ## verdict ## ]]\n' + 'yes\n' + '\n' + '[[ ## confidence ## ]]\n' + '0.8\n' + '\n' + '[[ ## completed ## ]]\n'}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, + {"type": "text", "text": "\n\n[[ ## audio ## ]]\n"}, + {"type": "input_audio", "input_audio": {"data": "Q1VSUkVOVA==", "format": "wav"}}, + {"type": "text", "text": "\n\n[[ ## file ## ]]\n"}, + {"type": "file", "file": {"file_id": "file-current", "filename": "current.txt"}}, + {"type": "text", "text": "\n\n[[ ## document ## ]]\n"}, + {"type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": "Current document"}, + "citations": {"enabled": True}, + "title": "Current Doc"}, + {"type": "text", "text": "\n\n[[ ## event ## ]]\n"}, + {"type": "event", "event": {"label": "current-event"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' + '"interests": ["compilers", "navy"]}\n' + '\n' + '[[ ## context ## ]]\n' + '[1] «current context one»\n' + '[2] «current context two»\n' + '\n' + '[[ ## question ## ]]\n' + 'What should the answer include?\n' + '\n' + 'Respond with the corresponding output fields, starting with the field `[[ ' + '## answer ## ]]` (must be formatted as a valid Python AnswerCard), then ' + "`[[ ## verdict ## ]]` (must be formatted as a valid Python Literal['yes', " + "'no']), then `[[ ## confidence ## ]]` (must be formatted as a valid Python " + 'float), and then ending with the marker for `[[ ## completed ## ]]`.'}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + def test_chat_adapter_with_pydantic_models(): """ This test verifies that ChatAdapter can handle different input and output field types, both basic and nested. @@ -195,7 +1531,7 @@ def test_chat_adapter_exception_raised_on_failure(): signature = dspy.make_signature("question->answer") adapter = dspy.ChatAdapter() invalid_completion = "{'output':'mismatched value'}" - with pytest.raises(dspy.utils.exceptions.AdapterParseError, match="Adapter ChatAdapter failed to parse*"): + with pytest.raises(dspy.utils.exceptions.AdapterParseError, match=r"Adapter ChatAdapter failed to parse.*"): adapter.parse(signature, invalid_completion) @@ -562,7 +1898,13 @@ def get_weather(city: str) -> str: ) assert result[0]["tool_calls"] == dspy.ToolCalls( - tool_calls=[dspy.ToolCalls.ToolCall(name="get_weather", args={"city": "Paris"})] + tool_calls=[ + dspy.ToolCalls.ToolCall( + name="get_weather", + args={"city": "Paris"}, + id="call_pQm8ajtSMxgA0nrzK2ivFmxG", + ) + ] ) # `answer` is not present, so we set it to None assert result[0]["answer"] is None @@ -584,6 +1926,124 @@ def get_weather(city: str) -> str: assert result[0]["tool_calls"] is None +def test_chat_adapter_toolcalls_serial_non_native_schema_limits_cardinality(): + class MySignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + def get_weather(city: str) -> str: + return f"The weather in {city} is sunny" + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(allow_parallel_tool_calls=False), + MySignature, + [], + {"question": "What is the weather in Paris?", "tools": [dspy.Tool(get_weather)]}, + ) + + assert '"maxItems": 1' in messages[0]["content"] + assert "parallel_tool_calls" not in lm_kwargs + + +def test_chat_adapter_toolcalls_native_serial_forwards_provider_flag(): + class MySignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + def get_weather(city: str) -> str: + return f"The weather in {city} is sunny" + + adapter = dspy.ChatAdapter(use_native_function_calling=True, allow_parallel_tool_calls=False) + inputs = {"question": "What is the weather in Paris?", "tools": [dspy.Tool(get_weather)]} + lm = dspy.LM(model="openai/gpt-4o-mini", cache=False) + rendered = adapter._render_request(lm, {}, MySignature, [], inputs) + rendered_again = adapter._render_request(lm, {}, MySignature, [], inputs) + + assert rendered.request.config.tool_choice.parallel is False + assert rendered.request.config.extensions == {} + assert rendered.call_plan.tools + assert rendered.call_plan is not rendered_again.call_plan + assert not hasattr(rendered.context, "extra") + + _, lm_kwargs = format_messages_and_lm_kwargs( + adapter, + MySignature, + [], + inputs, + lm=lm, + ) + + assert lm_kwargs["parallel_tool_calls"] is False + + +def test_chat_adapter_toolcalls_native_render_uses_tool_call_path(): + class MySignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + def get_weather(city: str) -> str: + return f"The weather in {city} is sunny" + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.ChatAdapter(use_native_function_calling=True), + MySignature, + [], + {"question": "What is the weather in Paris?", "tools": [dspy.Tool(get_weather)]}, + lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), + ) + + assert "[[ ## tool_calls ## ]]" not in messages[0]["content"] + assert "[[ ## tools ## ]]" not in messages[0]["content"] + assert lm_kwargs["tools"][0]["function"]["name"] == "get_weather" + + +def test_chat_adapter_toolcalls_serial_non_native_rejects_multiple_calls(): + class MySignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + def get_weather(city: str) -> str: + return f"The weather in {city} is sunny" + + def get_time(city: str) -> str: + return f"The time in {city} is noon" + + adapter = dspy.ChatAdapter(allow_parallel_tool_calls=False, use_json_adapter_fallback=False) + with mock.patch("litellm.completion") as mock_completion: + mock_completion.return_value = ModelResponse( + choices=[ + Choices( + message=Message( + content=( + "[[ ## tool_calls ## ]]\n" + "[" + "{'name': 'get_weather', 'args': {'city': 'Paris'}}, " + "{'name': 'get_time', 'args': {'city': 'Paris'}}" + "]" + ) + ) + ) + ], + model="openai/gpt-4o-mini", + ) + + with pytest.raises(ValueError, match="permits at most 1"): + adapter( + dspy.LM(model="openai/gpt-4o-mini", cache=False), + {}, + MySignature, + [], + { + "question": "What is the weather and time in Paris?", + "tools": [dspy.Tool(get_weather), dspy.Tool(get_time)], + }, + ) + + def test_chat_adapter_toolcalls_vague_match(): class MySignature(dspy.Signature): question: str = dspy.InputField() @@ -664,13 +2124,14 @@ class MySignature(dspy.Signature): ], model="anthropic/claude-3-7-sonnet-20250219", ) - modified_signature = adapter._call_preprocess( - dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), - {}, + messages, _ = format_messages_and_lm_kwargs( + adapter, MySignature, + [], {"question": "What is the capital of France?"}, + lm=dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), ) - assert "reasoning" not in modified_signature.output_fields + assert "[[ ## reasoning ## ]]" not in messages[0]["content"] result = adapter( dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), @@ -726,7 +2187,7 @@ class MySignature(dspy.Signature): expected_system_message = """Your input fields are: 1. `question` (str): Your output fields are: -1. `answers` (list[str]): +1. `answers` (list[str]):\x20 2. `scores` (list[float]): All interactions will be structured in the following way, with the appropriate values filled in. @@ -740,7 +2201,7 @@ class MySignature(dspy.Signature): {scores} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "number"}} [[ ## completed ## ]] -In adhering to this structure, your objective is: +In adhering to this structure, your objective is:\x20 Answer the question with multiple answers and scores""" assert system_message == expected_system_message @@ -781,15 +2242,23 @@ def test_empty_string_content_raises_adapter_parse_error(): def test_tool_call_with_null_content_does_not_raise(): - """Tool-call-only responses legitimately have content=None. - _call_postprocess must NOT raise when tool_calls are present.""" + """Tool-call-only responses legitimately have content=None and should parse.""" + from dspy.clients.openai_format import lm_response_from_legacy_outputs + adapter = dspy.ChatAdapter(use_native_function_calling=True) sig_cls = dspy.Signature("question, tools: list[dspy.Tool] -> answer, tool_calls: dspy.ToolCalls") + def search(query: str) -> str: + return query + + inputs = {"question": "test", "tools": [dspy.Tool(search)]} + lm = dspy.LM("openai/gpt-4o-mini", cache=False) + rendered = adapter._render_request(lm, {}, sig_cls, [], inputs) outputs = [{"text": None, "tool_calls": [ {"function": {"name": "search", "arguments": '{"query": "test"}'}, "id": "call_1", "type": "function"} ]}] + response = lm_response_from_legacy_outputs(outputs, rendered.request) - result = adapter._call_postprocess(sig_cls, sig_cls, outputs, None, {}) + result = adapter._parse_response(sig_cls, rendered.call_plan, response, rendered.context) assert result is not None assert len(result) == 1 diff --git a/tests/adapters/test_citation.py b/tests/adapters/test_citation.py index 3c2a7f75e6..a7bed4372e 100644 --- a/tests/adapters/test_citation.py +++ b/tests/adapters/test_citation.py @@ -2,6 +2,9 @@ import pytest import dspy +from dspy.adapters._type_feature_handlers import _CitationsTypeHandler +from dspy.adapters._type_runtime import _AdapterCallPlan +from dspy.core.types import LMCitationPart, LMOutput from dspy.experimental import Citations @@ -106,6 +109,52 @@ def test_citations_format(): assert formatted[1]["document_title"] == "Source" +def test_citations_type_handler_uses_native_lm_output_and_parts(): + class CitationSignature(dspy.Signature): + question: str = dspy.InputField() + citations: Citations = dspy.OutputField() + + adapter = dspy.Adapter(native_response_types=[Citations]) + lm = dspy.LM(model="anthropic/claude-3-5-sonnet-20241022") + context = adapter.build_call_context(lm) + call = _AdapterCallPlan.from_signature(CitationSignature, {"question": "Q?"}, {}) + type_handler = _CitationsTypeHandler() + type_handler.prepare(call, context) + + citation = Citations.Citation( + cited_text="The sky is blue", + document_index=0, + document_title="Weather Guide", + start_char_index=10, + end_char_index=25, + supported_text="The sky is blue", + ) + rendered = type_handler.format_output("citations", Citations(citations=[citation]), call, context) + + assert "citations" not in call.render_signature.output_fields + assert rendered.parts[0].text == "The sky is blue" + + values = {} + type_handler.parse( + values, + LMOutput( + parts=[ + LMCitationPart( + text="The sky is blue", + title="Weather Guide", + metadata={"document_index": 0, "start_char_index": 10, "end_char_index": 25}, + ) + ] + ), + call, + context, + ) + + parsed = values["citations"] + assert parsed.citations[0].cited_text == "The sky is blue" + assert parsed.citations[0].document_title == "Weather Guide" + + def test_citations_from_dict_list(): citations_data = [ { @@ -151,13 +200,12 @@ class CitationSignature(Signature): ] }] - result = adapter._call_postprocess( - CitationSignature.delete("citations"), - CitationSignature, - outputs, - dspy.LM(model="anthropic/claude-3-5-sonnet-20241022"), - lm_kwargs={}, - ) + from dspy.clients.openai_format import lm_response_from_legacy_outputs + + lm = dspy.LM(model="anthropic/claude-3-5-sonnet-20241022") + rendered = adapter._render_request(lm, {}, CitationSignature, [], {"question": "What color is the sky?"}) + response = lm_response_from_legacy_outputs(outputs, rendered.request) + result = adapter._parse_response(CitationSignature, rendered.call_plan, response, rendered.context) assert len(result) == 1 assert "citations" in result[0] diff --git a/tests/adapters/test_history.py b/tests/adapters/test_history.py new file mode 100644 index 0000000000..fee489d468 --- /dev/null +++ b/tests/adapters/test_history.py @@ -0,0 +1,115 @@ +from dspy.adapters.types.history import History, make_truncate_oldest_actions, truncate_oldest_actions +from dspy.adapters.types.tool import ToolCallResults, ToolCalls + + +def test_messages_key_is_canonical_history_shape(): + legacy_message = {"question": "What is the capital of France?", "answer": "Paris"} + + history = History(messages=[legacy_message]) + + assert history.messages[0] == legacy_message + assert history.model_dump() == {"messages": [legacy_message]} + + +def test_append_adds_plain_messages(): + tool_calls = ToolCalls.from_dict_list([{"name": "search", "args": {"query": "hello"}, "id": "call_0"}]) + tool_results = ToolCallResults.from_dict_list( + [{"call_id": "call_0", "name": "search", "value": "result", "is_error": False}] + ) + history = History(messages=[]) + + history.append( + { + "question": "hi", + "next_thought": "search first", + "tool_calls": tool_calls, + "tool_call_results": tool_results, + "answer": "bye", + } + ) + + assert history.messages == [ + { + "question": "hi", + "next_thought": "search first", + "tool_calls": tool_calls, + "tool_call_results": tool_results, + "answer": "bye", + }, + ] + assert History.model_validate(history.model_dump()) == history + + +def test_history_normalizes_raw_tool_call_lists_on_canonical_keys(): + history = History( + messages=[ + {"tool_calls": [{"name": "search", "args": {"query": "hello"}, "id": "call_0"}]}, + {"tool_call_results": [{"call_id": "call_0", "name": "search", "value": "result"}]}, + ] + ) + + assert history.messages[0]["tool_calls"] == ToolCalls.from_dict_list( + [{"name": "search", "args": {"query": "hello"}, "id": "call_0"}] + ) + assert history.messages[1]["tool_call_results"] == ToolCallResults.from_dict_list( + [{"call_id": "call_0", "name": "search", "value": "result"}] + ) + + +def test_compact_if_needed_calls_compact_fn_with_history(): + calls = [] + history = History(messages=[], compact_fn=calls.append) + + history.compact_if_needed() + + assert calls == [history] + + +def test_truncate_oldest_actions_keeps_recent_complete_tool_turns_and_non_actions(): + history = History(messages=[{"question": "new"}, {"question": "legacy", "answer": "legacy answer"}]) + + for index in range(5): + history.append( + { + "next_thought": str(index), + "tool_calls": ToolCalls.from_dict_list( + [{"name": "search", "args": {"query": str(index)}, "id": f"call_{index}"}] + ), + "tool_call_results": ToolCallResults.from_dict_list( + [{"call_id": f"call_{index}", "name": "search", "value": f"result {index}"}] + ), + } + ) + history.append({"answer": "done"}) + + truncate_oldest_actions(history, max_tokens=0, keep_n=2) + + action_messages = [message for message in history.messages if "tool_calls" in message] + result_messages = [message for message in history.messages if "tool_call_results" in message] + assert [message["next_thought"] for message in action_messages] == ["3", "4"] + assert [message["tool_call_results"].tool_call_results[0].value for message in result_messages] == [ + "result 3", + "result 4", + ] + assert history.messages[0] == {"question": "new"} + assert history.messages[1] == {"question": "legacy", "answer": "legacy answer"} + assert history.messages[-1] == {"answer": "done"} + + +def test_make_truncate_oldest_actions_returns_compaction_fn(): + history = History(messages=[]) + for index in range(4): + history.append( + { + "next_thought": str(index), + "tool_calls": ToolCalls.from_dict_list([{"name": "search", "args": {}, "id": f"call_{index}"}]), + "tool_call_results": ToolCallResults.from_dict_list( + [{"call_id": f"call_{index}", "name": "search", "value": f"result {index}"}] + ), + } + ) + + make_truncate_oldest_actions(max_tokens=0, keep_n=1)(history) + + action_messages = [message for message in history.messages if "tool_calls" in message] + assert [message["next_thought"] for message in action_messages] == ["3"] diff --git a/tests/adapters/test_history_formatting.py b/tests/adapters/test_history_formatting.py new file mode 100644 index 0000000000..51c141e63c --- /dev/null +++ b/tests/adapters/test_history_formatting.py @@ -0,0 +1,72 @@ +import json + +import dspy +from dspy.adapters.types.history import History + + +def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + +class AgentTurn(dspy.Signature): + question: str = dspy.InputField() + history: dspy.History = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_call_results: dspy.ToolCallResults = dspy.InputField() + next_thought: str = dspy.OutputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + +def _history_with_complete_tool_turn() -> History: + history = History(messages=[]) + history.append( + { + "question": "What is 1+2?", + "next_thought": "I should add.", + "tool_calls": dspy.ToolCalls.from_dict_list([{"name": "add", "args": {"a": 1, "b": 2}}]), + "tool_call_results": dspy.ToolCallResults.from_dict_list( + [{"name": "add", "value": 3, "is_error": False}] + ), + } + ) + return history + + +def test_chat_adapter_formats_complete_tool_turn_without_duplicate_input(): + messages = dspy.ChatAdapter().format( + AgentTurn, + [], + {"history": _history_with_complete_tool_turn(), "tools": [dspy.Tool(add)]}, + ) + + texts = [message["content"] or "" for message in messages] + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "user", "user"] + assert sum("What is 1+2?" in text for text in texts[1:]) == 1 + assert "[[ ## next_thought ## ]]\nI should add." in texts[2] + assert "[[ ## tool_calls ## ]]" in texts[2] + assert "[[ ## tool_call_results ## ]]" in texts[3] + assert '"name": "add"' in texts[3] + assert '"value": 3' in texts[3] + assert "Respond with the corresponding output fields" in texts[4] + + +def test_json_adapter_formats_complete_tool_turn_without_duplicate_input(): + messages = dspy.JSONAdapter().format( + AgentTurn, + [], + {"history": _history_with_complete_tool_turn(), "tools": [dspy.Tool(add)]}, + ) + + texts = [message["content"] or "" for message in messages] + assistant = json.loads(texts[2]) + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "user", "user"] + assert sum("What is 1+2?" in text for text in texts[1:]) == 1 + assert assistant["next_thought"] == "I should add." + assert assistant["tool_calls"]["tool_calls"][0]["name"] == "add" + assert "[[ ## tool_call_results ## ]]" in texts[3] + assert '"name": "add"' in texts[3] + assert '"value": 3' in texts[3] + assert "Respond with a JSON object" in texts[4] diff --git a/tests/adapters/test_history_lm_messages.py b/tests/adapters/test_history_lm_messages.py new file mode 100644 index 0000000000..66867dd6ab --- /dev/null +++ b/tests/adapters/test_history_lm_messages.py @@ -0,0 +1,95 @@ +import json + +import dspy +from dspy.adapters.types.history import History +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls + + +def add(a: int, b: int) -> int: + return a + b + + +class NativeToolLM: + model = "openai/gpt-5-nano" + supported_params = frozenset() + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + + def __call__(self, messages, **kwargs): + self.messages = messages + self.kwargs = kwargs + return [{"text": "[[ ## next_thought ## ]]\nDone.\n\n[[ ## completed ## ]]"}] + + +def test_history_renders_native_tool_messages_through_lm_message_path(): + adapter = dspy.ChatAdapter(use_native_function_calling=True) + signature = ( + dspy.Signature({}, "Do the task.") + .append("question", dspy.InputField(), type_=str) + .append("history", dspy.InputField(), type_=dspy.History) + .append("tools", dspy.InputField(), type_=list[dspy.Tool]) + .append("previous_tool_results", dspy.InputField(), type_=dspy.ToolCallResults) + .append("next_thought", dspy.OutputField(), type_=str) + .append("tool_calls", dspy.OutputField(), type_=dspy.ToolCalls) + ) + history = History(messages=[]) + history.append( + { + "question": "What is 1+2?", + "next_thought": "I should add the two numbers.", + "tool_calls": ToolCalls.from_dict_list( + [{"name": "add", "args": {"a": 1, "b": 2}, "id": "call_add"}] + ), + "previous_tool_results": ToolCallResults.from_dict_list( + [{"call_id": "call_add", "name": "add", "value": 3}] + ), + } + ) + + lm = NativeToolLM() + adapter(lm, {}, signature, [], {"history": history, "tools": [Tool(add)]}) + + assistant_message = next(message for message in lm.messages if message["role"] == "assistant") + tool_message = next(message for message in lm.messages if message["role"] == "tool") + final_user_message = lm.messages[-1] + + assert assistant_message["content"] == "I should add the two numbers." + assert assistant_message["tool_calls"][0]["id"] == "call_add" + assert assistant_message["tool_calls"][0]["function"]["name"] == "add" + assert json.loads(assistant_message["tool_calls"][0]["function"]["arguments"]) == {"a": 1, "b": 2} + assert tool_message == {"role": "tool", "content": "3", "tool_call_id": "call_add", "name": "add"} + assert "What is 1+2?" not in final_user_message["content"] + assert "Respond with the corresponding output fields" in final_user_message["content"] + + +def test_current_tool_result_value_renders_native_tool_messages_independent_of_field_name(): + adapter = dspy.ChatAdapter(use_native_function_calling=True) + signature = ( + dspy.Signature({}, "Continue the task.") + .append("question", dspy.InputField(), type_=str) + .append("tools", dspy.InputField(), type_=list[dspy.Tool]) + .append("previous_tool_results", dspy.InputField(), type_=dspy.ToolCallResults) + .append("next_thought", dspy.OutputField(), type_=str) + .append("tool_calls", dspy.OutputField(), type_=dspy.ToolCalls) + ) + tool_results = ToolCallResults.from_dict_list([{"call_id": "call_add", "name": "add", "value": 3}]) + + lm = NativeToolLM() + adapter( + lm, + {}, + signature, + [], + { + "question": "What is 1+2?", + "tools": [Tool(add)], + "previous_tool_results": tool_results, + }, + ) + + tool_message = next(message for message in lm.messages if message["role"] == "tool") + user_messages = [message for message in lm.messages if message["role"] == "user"] + + assert tool_message == {"role": "tool", "content": "3", "tool_call_id": "call_add", "name": "add"} + assert all("previous_tool_results" not in (message["content"] or "") for message in user_messages) diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py index f376831d98..e0f9851334 100644 --- a/tests/adapters/test_json_adapter.py +++ b/tests/adapters/test_json_adapter.py @@ -1,3 +1,5 @@ +import enum +from typing import Literal from unittest import mock import pydantic @@ -7,6 +9,653 @@ from openai.types.responses import ResponseOutputMessage import dspy +from tests.adapters.conftest import format_messages_and_lm_kwargs + + +def test_json_adapter_format_exact_messages_for_simple_signature(): + class StringSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), + StringSignature, + demos=[], + inputs={"question": "What is the capital of France?"}, + ) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str): +All interactions will be structured in the following way, with the appropriate values filled in. + +Inputs will have the following structure: + +[[ ## question ## ]] +{question} + +Outputs will be a JSON object with the following fields. + +{ + "answer": "{answer}" +} +In adhering to this structure, your objective is:\x20 + Given the fields `question`, produce the fields `answer`.""", + }, + { + "role": "user", + "content": """[[ ## question ## ]] +What is the capital of France? + +Respond with a JSON object in the following order of fields: `answer`.""", + }, + ] + + +def test_json_adapter_format_exact_messages_with_demo_and_typed_output(): + class MultiAnswer(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + confidence: float = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), + MultiAnswer, + demos=[{"question": "Q1", "answer": "A1", "confidence": 0.9}], + inputs={"question": "Q2"}, + ) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str):\x20 +2. `confidence` (float): +All interactions will be structured in the following way, with the appropriate values filled in. + +Inputs will have the following structure: + +[[ ## question ## ]] +{question} + +Outputs will be a JSON object with the following fields. + +{ + "answer": "{answer}", + "confidence": "{confidence} # note: the value you produce must be a single float value" +} +In adhering to this structure, your objective is:\x20 + Given the fields `question`, produce the fields `answer`, `confidence`.""", + }, + {"role": "user", "content": """[[ ## question ## ]] +Q1"""}, + { + "role": "assistant", + "content": """{ + "answer": "A1", + "confidence": 0.9 +}""", + }, + { + "role": "user", + "content": """[[ ## question ## ]] +Q2 + +Respond with a JSON object in the following order of fields: `answer`, then `confidence` (must be formatted as a valid Python float).""", + }, + ] + + +def test_json_adapter_format_exact_messages_with_described_and_bool_outputs(): + class TestSignature(dspy.Signature): + input1: str = dspy.InputField() + output1: str = dspy.OutputField(desc="String output field") + output2: bool = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), TestSignature, [], {"input1": "Test input"}) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `input1` (str): +Your output fields are: +1. `output1` (str): String output field +2. `output2` (bool): +All interactions will be structured in the following way, with the appropriate values filled in. + +Inputs will have the following structure: + +[[ ## input1 ## ]] +{input1} + +Outputs will be a JSON object with the following fields. + +{ + "output1": "{output1}", + "output2": "{output2} # note: the value you produce must be True or False" +} +In adhering to this structure, your objective is:\x20 + Given the fields `input1`, produce the fields `output1`, `output2`.""", + }, + { + "role": "user", + "content": """[[ ## input1 ## ]] +Test input + +Respond with a JSON object in the following order of fields: `output1`, then `output2` (must be formatted as a valid Python bool).""", + }, + ] + + +def test_json_adapter_format_exact_messages_with_history_demo_pydantic_tools_and_image(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class Location(pydantic.BaseModel): + city: str + country: str + + class Profile(pydantic.BaseModel): + name: str + location: Location + interests: list[str] + + class AnswerCard(pydantic.BaseModel): + answer: str + sources: list[str] + + class RichRenderingSignature(dspy.Signature): + """Answer using all supplied context.""" + + history: dspy.History = dspy.InputField() + image: dspy.Image = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + profile: Profile = dspy.InputField() + question: str = dspy.InputField() + answer: AnswerCard = dspy.OutputField() + + tool = dspy.Tool(search) + demo_profile = Profile( + name="Ada", + location=Location(city="London", country="UK"), + interests=["math", "machines"], + ) + current_profile = Profile( + name="Grace", + location=Location(city="Arlington", country="USA"), + interests=["compilers", "navy"], + ) + history = dspy.History( + messages=[ + { + "profile": demo_profile, + "question": "Who is Ada?", + "answer": AnswerCard(answer="Ada is a mathematician.", sources=["memory"]), + } + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), + RichRenderingSignature, + demos=[ + { + "image": dspy.Image("https://example.com/demo.png"), + "tools": [tool], + "profile": demo_profile, + "question": "What should we mention?", + "answer": AnswerCard(answer="Mention analytical engines.", sources=["demo"]), + } + ], + inputs={ + "history": history, + "image": dspy.Image("https://example.com/current.png"), + "tools": [tool], + "profile": current_profile, + "question": "What should the answer include?", + }, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `image` (Image): \n' + '2. `tools` (list[Tool]): \n' + '3. `profile` (Profile): \n' + '4. `question` (str):\n' + 'Your output fields are:\n' + '1. `answer` (AnswerCard):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## image ## ]]\n' + '{image}\n' + '\n' + '[[ ## tools ## ]]\n' + '{tools}\n' + '\n' + '[[ ## profile ## ]]\n' + '{profile}\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "answer": "{answer} # note: the value you produce must adhere to the JSON ' + 'schema: {\\"type\\": \\"object\\", \\"properties\\": {\\"answer\\": {\\"type\\": ' + '\\"string\\", \\"title\\": \\"Answer\\"}, \\"sources\\": {\\"type\\": \\"array\\", ' + '\\"items\\": {\\"type\\": \\"string\\"}, \\"title\\": \\"Sources\\"}}, ' + '\\"required\\": [\\"answer\\", \\"sources\\"], \\"title\\": \\"AnswerCard\\"}"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Answer using all supplied context.'}, + {"role": "user", + "content": [{"type": "text", "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It takes ' + "arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', 'default': 3}}.\"]\n" + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' + '"interests": ["math", "machines"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'What should we mention?'}]}, + {"role": "assistant", + "content": '{\n' + ' "answer": {\n' + ' "answer": "Mention analytical engines.",\n' + ' "sources": [\n' + ' "demo"\n' + ' ]\n' + ' }\n' + '}'}, + {"role": "user", + "content": '[[ ## profile ## ]]\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, "interests": ' + '["math", "machines"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'Who is Ada?'}, + {"role": "assistant", + "content": '{\n' + ' "answer": {\n' + ' "answer": "Ada is a mathematician.",\n' + ' "sources": [\n' + ' "memory"\n' + ' ]\n' + ' }\n' + '}'}, + {"role": "user", + "content": [{"type": "text", + "text": "[[ ## image ## ]]\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, + {"type": "text", + "text": '\n' + '\n' + '[[ ## tools ## ]]\n' + '["search, whose description is Search for documents.. It takes ' + "arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', 'default': 3}}.\"]\n" + '\n' + '[[ ## profile ## ]]\n' + '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' + '"interests": ["compilers", "navy"]}\n' + '\n' + '[[ ## question ## ]]\n' + 'What should the answer include?\n' + '\n' + 'Respond with a JSON object in the following order of fields: `answer` ' + '(must be formatted as a valid Python AnswerCard).'}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_json_adapter_format_exact_messages_with_int_and_mapping_outputs(): + class IntDictSignature(dspy.Signature): + question: str = dspy.InputField() + count: int = dspy.OutputField() + metadata: dict[str, int] = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), IntDictSignature, [], {"question": "Count things"}) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `count` (int): \n' + '2. `metadata` (dict[str, int]):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "count": "{count} # note: the value you produce must be a single int ' + 'value",\n' + ' "metadata": "{metadata} # note: the value you produce must adhere to the ' + 'JSON schema: {\\"type\\": \\"object\\", \\"additionalProperties\\": {\\"type\\": ' + '\\"integer\\"}}"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `count`, `metadata`.'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Count things\n" + "\n" + "Respond with a JSON object in the following order of fields: `count` (must be " + "formatted as a valid Python int), then `metadata` (must be formatted as a valid " + "Python dict[str, int])."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_json_adapter_format_exact_messages_with_literal_and_enum_outputs(): + class Label(enum.Enum): + POSITIVE = "positive" + NEGATIVE = "negative" + + class LiteralEnumSignature(dspy.Signature): + text: str = dspy.InputField() + decision: Literal["accept", "reject"] = dspy.OutputField() + label: Label = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), LiteralEnumSignature, [], {"text": "Looks good"}) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `text` (str):\n' + 'Your output fields are:\n' + "1. `decision` (Literal['accept', 'reject']): \n" + '2. `label` (Label):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## text ## ]]\n' + '{text}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "decision": "{decision} # note: the value you produce must exactly match ' + '(no extra characters) one of: accept; reject",\n' + ' "label": "{label} # note: the value you produce must be one of: positive; ' + 'negative"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `text`, produce the fields `decision`, `label`.'}, + {"role": "user", + "content": "[[ ## text ## ]]\n" + "Looks good\n" + "\n" + "Respond with a JSON object in the following order of fields: `decision` (must be " + "formatted as a valid Python Literal['accept', 'reject']), then `label` (must be " + "formatted as a valid Python Label)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_json_adapter_format_exact_messages_with_nested_pydantic_output(): + class JsonNestedAddress(pydantic.BaseModel): + city: str + country: str + + class JsonNestedSummary(pydantic.BaseModel): + title: str + address: JsonNestedAddress + scores: list[float] + + class PydanticSignature(dspy.Signature): + question: str = dspy.InputField() + summary: JsonNestedSummary = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), PydanticSignature, [], {"question": "Summarize"}) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `summary` (JsonNestedSummary):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "summary": "{summary} # note: the value you produce must adhere to the JSON ' + 'schema: {\\"type\\": \\"object\\", \\"$defs\\": {\\"JsonNestedAddress\\": ' + '{\\"type\\": \\"object\\", \\"properties\\": {\\"city\\": {\\"type\\": \\"string\\", ' + '\\"title\\": \\"City\\"}, \\"country\\": {\\"type\\": \\"string\\", \\"title\\": ' + '\\"Country\\"}}, \\"required\\": [\\"city\\", \\"country\\"], \\"title\\": ' + '\\"JsonNestedAddress\\"}}, \\"properties\\": {\\"address\\": {\\"$ref\\": ' + '\\"#/$defs/JsonNestedAddress\\"}, \\"scores\\": {\\"type\\": \\"array\\", ' + '\\"items\\": {\\"type\\": \\"number\\"}, \\"title\\": \\"Scores\\"}, \\"title\\": ' + '{\\"type\\": \\"string\\", \\"title\\": \\"Title\\"}}, \\"required\\": [\\"title\\", ' + '\\"address\\", \\"scores\\"], \\"title\\": \\"JsonNestedSummary\\"}"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `summary`.'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Summarize\n" + "\n" + "Respond with a JSON object in the following order of fields: `summary` (must be " + "formatted as a valid Python JsonNestedSummary)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_json_adapter_format_exact_messages_with_incomplete_demo(): + class IncompleteDemoSignature(dspy.Signature): + question: str = dspy.InputField() + context: str = dspy.InputField() + answer: str = dspy.OutputField() + score: float = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(), + IncompleteDemoSignature, + [{"question": "Q1", "answer": "A1"}], + {"question": "Q2", "context": "C2"}, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str): \n' + '2. `context` (str):\n' + 'Your output fields are:\n' + '1. `answer` (str): \n' + '2. `score` (float):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + '[[ ## context ## ]]\n' + '{context}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "answer": "{answer}",\n' + ' "score": "{score} # note: the value you produce must be a single float ' + 'value"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, `context`, produce the fields `answer`, ' + '`score`.'}, + {"role": "user", + "content": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "[[ ## question ## ]]\n" + "Q1"}, + {"role": "assistant", + "content": '{\n "answer": "A1",\n "score": "Not supplied for this particular example. "\n}'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "[[ ## context ## ]]\n" + "C2\n" + "\n" + "Respond with a JSON object in the following order of fields: `answer`, then `score` " + "(must be formatted as a valid Python float)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_json_adapter_format_exact_messages_and_lm_kwargs_with_native_tool_calling(): + class FunctionCallingLM(dspy.utils.DummyLM): + @property + def supports_function_calling(self): + return True + + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class NativeToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs( + dspy.JSONAdapter(use_native_function_calling=True), + NativeToolSignature, + [], + {"question": "Q?", "tools": [dspy.Tool(search)]}, + lm=FunctionCallingLM([{}]), + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "Inputs will have the following structure:\n" + "\n" + "[[ ## question ## ]]\n" + "{question}\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `tools`, produce the fields `tool_calls`."}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q?"}] + assert messages == expected_messages + expected_lm_kwargs = {"tools": [{"type": "function", + "function": {"name": "search", + "description": "Search for documents.", + "parameters": {"type": "object", + "properties": {"query": {"type": "string"}, + "k": {"type": "integer", "default": 3}}, + "required": ["query", "k"]}}}]} + assert lm_kwargs == expected_lm_kwargs + +def test_json_adapter_format_exact_messages_with_tool_calls_output_demo(): + class ToolCallsSignature(dspy.Signature): + question: str = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.JSONAdapter(use_native_function_calling=False), + ToolCallsSignature, + [{"question": "Q1", "tool_calls": dspy.ToolCalls.from_dict_list([{"name": "search", "args": {"query": "cats"}}])}], + {"question": "Q2"}, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `tool_calls` (ToolCalls): \n' + ' Type description of ToolCalls: Tool calls information, including the name of the ' + 'tools and the arguments to be passed to it. Arguments must be provided in JSON ' + 'format.\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + 'Inputs will have the following structure:\n' + '\n' + '[[ ## question ## ]]\n' + '{question}\n' + '\n' + 'Outputs will be a JSON object with the following fields.\n' + '\n' + '{\n' + ' "tool_calls": "{tool_calls} # note: the value you produce must adhere to ' + 'the JSON schema: {\\"type\\": \\"object\\", \\"$defs\\": {\\"ToolCall\\": ' + '{\\"type\\": \\"object\\", \\"properties\\": {\\"args\\": {\\"type\\": \\"object\\", ' + '\\"additionalProperties\\": true, \\"title\\": \\"Args\\"}, \\"id\\": {\\"anyOf\\": ' + '[{\\"type\\": \\"string\\"}, {\\"type\\": \\"null\\"}], \\"default\\": null, ' + '\\"title\\": \\"Id\\"}, \\"name\\": {\\"type\\": \\"string\\", \\"title\\": ' + '\\"Name\\"}}, \\"required\\": [\\"name\\", \\"args\\"], ' + '\\"title\\": \\"ToolCall\\"}}, \\"properties\\": {\\"tool_calls\\": {\\"type\\": ' + '\\"array\\", \\"items\\": {\\"$ref\\": \\"#/$defs/ToolCall\\"}, \\"title\\": \\"Tool ' + 'Calls\\"}}, \\"required\\": [\\"tool_calls\\"], \\"title\\": \\"ToolCalls\\"}"\n' + '}\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `tool_calls`.'}, + {"role": "user", "content": "[[ ## question ## ]]\nQ1"}, + {"role": "assistant", + "content": '{\n' + ' "tool_calls": {\n' + ' "tool_calls": [\n' + ' {\n' + ' "name": "search",\n' + ' "args": {\n' + ' "query": "cats"\n' + ' }\n' + ' }\n' + ' ]\n' + ' }\n' + '}'}, + {"role": "user", + "content": "[[ ## question ## ]]\n" + "Q2\n" + "\n" + "Respond with a JSON object in the following order of fields: `tool_calls` (must be " + "formatted as a valid Python ToolCalls)."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs def test_json_adapter_passes_structured_output_when_supported_by_model(): @@ -833,7 +1482,13 @@ def get_weather(city: str) -> str: ) assert result[0]["tool_calls"] == dspy.ToolCalls( - tool_calls=[dspy.ToolCalls.ToolCall(name="get_weather", args={"city": "Paris"})] + tool_calls=[ + dspy.ToolCalls.ToolCall( + name="get_weather", + args={"city": "Paris"}, + id="call_pQm8ajtSMxgA0nrzK2ivFmxG", + ) + ] ) # `answer` is not present, so we set it to None assert result[0]["answer"] is None @@ -907,13 +1562,14 @@ class MySignature(dspy.Signature): ], model="anthropic/claude-3-7-sonnet-20250219", ) - modified_signature = adapter._call_preprocess( - dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), - {}, + messages, _ = format_messages_and_lm_kwargs( + adapter, MySignature, + [], {"question": "What is the capital of France?"}, + lm=dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), ) - assert "reasoning" not in modified_signature.output_fields + assert '"reasoning"' not in messages[0]["content"] result = adapter( dspy.LM(model="anthropic/claude-3-7-sonnet-20250219", reasoning_effort="low", cache=False), @@ -998,7 +1654,7 @@ class MySignature(dspy.Signature): expected_system_message = """Your input fields are: 1. `question` (str): Your output fields are: -1. `answers` (list[str]): +1. `answers` (list[str]):\x20 2. `scores` (list[float]): All interactions will be structured in the following way, with the appropriate values filled in. @@ -1013,6 +1669,6 @@ class MySignature(dspy.Signature): "answers": "{answers} # note: the value you produce must adhere to the JSON schema: {\\"type\\": \\"array\\", \\"items\\": {\\"type\\": \\"string\\"}}", "scores": "{scores} # note: the value you produce must adhere to the JSON schema: {\\"type\\": \\"array\\", \\"items\\": {\\"type\\": \\"number\\"}}" } -In adhering to this structure, your objective is: +In adhering to this structure, your objective is:\x20 Answer the question with multiple answers and scores""" assert system_message == expected_system_message diff --git a/tests/adapters/test_reasoning.py b/tests/adapters/test_reasoning.py index 90812b81cb..80143775b3 100644 --- a/tests/adapters/test_reasoning.py +++ b/tests/adapters/test_reasoning.py @@ -1,6 +1,9 @@ import pytest import dspy +from dspy.adapters._type_feature_handlers import _ReasoningTypeHandler +from dspy.adapters._type_runtime import _AdapterCallPlan +from dspy.core.types import LMOutput, LMThinkingPart def test_reasoning_basic_operations(): @@ -109,3 +112,33 @@ def test_reasoning_error_message(): with pytest.raises(AttributeError, match="`Reasoning` object has no attribute 'nonexistent_method'"): reasoning.nonexistent_method + + +def test_reasoning_type_handler_uses_native_lm_output_and_config(): + class ReasoningSignature(dspy.Signature): + question: str = dspy.InputField() + reasoning: dspy.Reasoning = dspy.OutputField() + + class NativeReasoningLM: + model = "anthropic/claude-3-7-sonnet" + model_type = "chat" + supports_function_calling = False + supports_reasoning = True + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + adapter = dspy.Adapter() + context = adapter.build_call_context(NativeReasoningLM()) + call = _AdapterCallPlan.from_signature(ReasoningSignature, {"question": "Q?"}, {}) + type_handler = _ReasoningTypeHandler() + + type_handler.prepare(call, context) + + assert "reasoning" not in call.render_signature.output_fields + assert call.config.reasoning.effort == "low" + + values = {} + type_handler.parse(values, LMOutput(parts=[LMThinkingPart(text="think")]), call, context) + + assert values["reasoning"] == dspy.Reasoning(content="think") diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py index cfcffe0947..e0f726dc83 100644 --- a/tests/adapters/test_tool.py +++ b/tests/adapters/test_tool.py @@ -5,7 +5,10 @@ from pydantic import BaseModel import dspy -from dspy.adapters.types.tool import Tool, ToolCalls, convert_input_schema_to_tool_args +from dspy.adapters._type_feature_handlers import _ToolTypeHandler +from dspy.adapters._type_runtime import _AdapterCallPlan +from dspy.adapters.types.tool import Tool, ToolCallResults, ToolCalls, convert_input_schema_to_tool_args +from dspy.core.types import LMOutput, LMToolCallPart # Test fixtures @@ -397,9 +400,7 @@ def test_async_tool_call_in_sync_mode(): ([], {"tool_calls": []}), ( [{"name": "search", "args": {"query": "hello"}}], - { - "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}], - }, + {"tool_calls": [{"name": "search", "args": {"query": "hello"}}]}, ), ( [ @@ -408,19 +409,14 @@ def test_async_tool_call_in_sync_mode(): ], { "tool_calls": [ - {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}, - { - "type": "function", - "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}}, - }, + {"name": "search", "args": {"query": "hello"}}, + {"name": "translate", "args": {"text": "world", "lang": "fr"}}, ], }, ), ( [{"name": "get_time", "args": {}}], - { - "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}], - }, + {"tool_calls": [{"name": "get_time", "args": {}}]}, ), ] @@ -446,8 +442,163 @@ def test_tool_calls_format_from_dict_list(): result = tool_calls.format() assert len(result["tool_calls"]) == 2 - assert result["tool_calls"][0]["function"]["name"] == "search" - assert result["tool_calls"][1]["function"]["name"] == "translate" + assert result["tool_calls"][0]["name"] == "search" + assert result["tool_calls"][1]["name"] == "translate" + + +def test_tool_calls_preserve_call_ids_and_fill_missing_ids(): + tool_calls = ToolCalls.from_dict_list( + [ + {"name": "search", "args": {"query": "hello"}, "id": "call_search"}, + {"name": "lookup", "args": {"key": "world"}}, + ] + ).with_call_ids("call") + + assert [tool_call.id for tool_call in tool_calls.tool_calls] == ["call_search", "call_1"] + assert [part.id for part in tool_calls.to_lm_parts()] == ["call_search", "call_1"] + + +def test_tool_calls_json_schema_can_limit_cardinality(): + schema = ToolCalls.json_schema(max_items=1) + + assert schema["properties"]["tool_calls"]["maxItems"] == 1 + + +def test_tool_calls_validate_max_items_rejects_too_many_calls(): + tool_calls = ToolCalls.from_dict_list( + [ + {"name": "search", "args": {"query": "hello"}}, + {"name": "lookup", "args": {"key": "world"}}, + ] + ) + + with pytest.raises(ValueError, match="permits at most 1"): + tool_calls.validate_max_items(1) + + +def test_native_tool_response_preserves_call_ids(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + answer: str = dspy.OutputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class ToolCallLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + + def __call__(self, messages, **kwargs): + return [ + { + "text": None, + "tool_calls": [ + { + "function": {"arguments": '{"x":1,"y":"two"}', "name": "dummy_function"}, + "id": "call_dummy", + "type": "function", + } + ], + } + ] + + adapter = dspy.ChatAdapter(use_native_function_calling=True) + result = adapter(ToolCallLM(), {}, ToolSignature, [], {"question": "call it", "tools": [Tool(dummy_function)]})[0] + + assert result["answer"] is None + assert result["tool_calls"] == ToolCalls.from_dict_list( + [{"name": "dummy_function", "args": {"x": 1, "y": "two"}, "id": "call_dummy"}] + ) + + +def test_tool_type_handler_renders_native_tool_specs_and_policy(): + class ToolSignature(dspy.Signature): + question: str = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + adapter = dspy.Adapter(use_native_function_calling=True, allow_parallel_tool_calls=False) + context = adapter.build_call_context(NativeToolLM()) + call = _AdapterCallPlan.from_signature( + ToolSignature, + {"question": "Q?", "tools": [Tool(dummy_function)]}, + {}, + ) + + _ToolTypeHandler().prepare(call, context) + + assert "tools" not in call.render_signature.input_fields + assert "tool_calls" not in call.render_signature.output_fields + assert call.config.tool_choice.parallel is False + assert call.tools[0].name == "dummy_function" + + +def test_toolcalls_parse_output_reads_native_lm_tool_calls(): + class ToolSignature(dspy.Signature): + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + adapter = dspy.Adapter(use_native_function_calling=True) + context = adapter.build_call_context(NativeToolLM()) + call = _AdapterCallPlan.from_signature(ToolSignature, {"tools": [Tool(dummy_function)]}, {}) + type_handler = _ToolTypeHandler() + type_handler.prepare(call, context) + output = LMOutput(parts=[LMToolCallPart(id="call_1", name="dummy_function", args={"x": 1, "y": "two"})]) + values = {} + + type_handler.parse(values, output, call, context) + + assert values["tool_calls"] == ToolCalls.from_dict_list( + [{"name": "dummy_function", "args": {"x": 1, "y": "two"}, "id": "call_1"}] + ) + + +def test_tool_result_type_hooks_render_native_tool_messages(): + class ToolResultSignature(dspy.Signature): + tools: list[dspy.Tool] = dspy.InputField() + result: dspy.ToolCallResults = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + adapter = dspy.Adapter(use_native_function_calling=True) + result = ToolCallResults.from_dict_list([{"call_id": "call_1", "name": "dummy_function", "value": "ok"}]) + context = adapter.build_call_context(NativeToolLM()) + call = _AdapterCallPlan.from_signature( + ToolResultSignature, + {"tools": [Tool(dummy_function)], "result": result}, + {}, + ) + + _ToolTypeHandler().prepare(call, context) + + assert "result" not in call.render_signature.input_fields + assert call.messages[0].role == "tool" + assert call.messages[0].parts[0].call_id == "call_1" def test_toolcalls_vague_match(): @@ -497,6 +648,47 @@ def test_toolcalls_vague_match(): ToolCalls.model_validate([{"foo": "bar"}]) +def test_toolcalls_accepts_normalized_lm_tool_call_parts(): + tool_calls = ToolCalls.model_validate([LMToolCallPart(id="call_1", name="search", args={"query": "hello"})]) + + assert tool_calls == ToolCalls.from_dict_list([{"id": "call_1", "name": "search", "args": {"query": "hello"}}]) + + +def test_toolcalls_rejects_provider_native_shapes(): + with pytest.raises(ValueError): + ToolCalls.model_validate( + [ + { + "type": "function", + "function": {"name": "search", "arguments": '{"query": "hello"}'}, + "id": "call_1", + } + ] + ) + + with pytest.raises(ValueError): + ToolCalls.model_validate({"type": "function_call", "name": "search", "arguments": '{"query": "hello"}'}) + + +def test_tool_call_results_use_canonical_shape(): + results = ToolCallResults.model_validate( + [{"call_id": "call_1", "name": "search", "value": {"answer": "hello"}, "is_error": False}] + ) + + assert results.format() == { + "tool_call_results": [ + {"call_id": "call_1", "name": "search", "value": {"answer": "hello"}, "is_error": False} + ] + } + + +def test_tool_call_results_reject_provider_native_shapes(): + with pytest.raises(ValueError): + ToolCallResults.model_validate( + {"type": "function_call_output", "call_id": "call_1", "output": '{"answer": "hello"}'} + ) + + def test_tool_convert_input_schema_to_tool_args_no_input_params(): args, arg_types, arg_desc = convert_input_schema_to_tool_args(schema={"properties": {}}) assert args == {} @@ -577,7 +769,7 @@ def get_pi(): tool_call4 = dspy.ToolCalls.ToolCall(name="nonexistent", args={}) try: tool_call4.execute(functions=tools) - assert False, "Should have raised ValueError" + raise AssertionError("Should have raised ValueError") except ValueError as e: assert "not found" in str(e) diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.py index f045188fbc..9bb5843fcc 100644 --- a/tests/adapters/test_two_step_adapter.py +++ b/tests/adapters/test_two_step_adapter.py @@ -3,6 +3,58 @@ import pytest import dspy +from tests.adapters.conftest import format_messages_and_lm_kwargs + + +def test_two_step_adapter_format_exact_messages_for_simple_signature_with_demo(): + class QA(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + adapter = dspy.TwoStepAdapter(dspy.utils.DummyLM([{"answer": "x"}])) + messages, lm_kwargs = format_messages_and_lm_kwargs(adapter, QA, [{"question": "Q1", "answer": "A1"}], {"question": "Q2"}) + + expected_messages = [{"role": "system", + "content": "You are a helpful assistant that can solve tasks based on user input.\n" + "As input, you will be provided with:\n" + "1. `question` (str):\n" + "Your outputs must contain:\n" + "1. `answer` (str):\n" + "You should lay out your outputs in detail so that your answer can be understood by " + "another agent\n" + "Specific instructions: Given the fields `question`, produce the fields `answer`."}, + {"role": "user", "content": "question: Q1"}, + {"role": "assistant", "content": "answer: A1"}, + {"role": "user", "content": "question: Q2"}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_two_step_adapter_format_exact_messages_with_typed_outputs(): + class TypedSignature(dspy.Signature): + question: str = dspy.InputField() + count: int = dspy.OutputField() + answer: str = dspy.OutputField() + + adapter = dspy.TwoStepAdapter(dspy.utils.DummyLM([{"count": 1, "answer": "x"}])) + messages, lm_kwargs = format_messages_and_lm_kwargs(adapter, TypedSignature, [], {"question": "Q"}) + + expected_messages = [{"role": "system", + "content": "You are a helpful assistant that can solve tasks based on user input.\n" + "As input, you will be provided with:\n" + "1. `question` (str):\n" + "Your outputs must contain:\n" + "1. `count` (int): \n" + "2. `answer` (str):\n" + "You should lay out your outputs in detail so that your answer can be understood by " + "another agent\n" + "Specific instructions: Given the fields `question`, produce the fields `count`, " + "`answer`."}, + {"role": "user", "content": "question: Q"}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs def test_two_step_adapter_call(): @@ -16,6 +68,7 @@ class TestSignature(dspy.Signature): mock_main_lm = mock.MagicMock(spec=dspy.LM) mock_main_lm.return_value = ["text from main LM"] mock_main_lm.kwargs = {"temperature": 1.0} + mock_main_lm.model = "openai/gpt-4o-mini" mock_extraction_lm = mock.MagicMock(spec=dspy.LM) mock_extraction_lm.return_value = [ @@ -82,6 +135,7 @@ class TestSignature(dspy.Signature): mock_main_lm = mock.MagicMock(spec=dspy.LM) mock_main_lm.acall.return_value = ["text from main LM"] mock_main_lm.kwargs = {"temperature": 1.0} + mock_main_lm.model = "openai/gpt-4o-mini" mock_extraction_lm = mock.MagicMock(spec=dspy.LM) mock_extraction_lm.acall.return_value = [ @@ -135,6 +189,44 @@ class TestSignature(dspy.Signature): assert "text from main LM" in content +@pytest.mark.asyncio +async def test_two_step_adapter_async_preserves_native_tool_call_ids(): + def add(a: int, b: int) -> int: + return a + b + + class ToolSignature(dspy.Signature): + tools: list[dspy.Tool] = dspy.InputField() + tool_calls: dspy.ToolCalls = dspy.OutputField() + + class NativeToolLM: + model = "openai/gpt-5-nano" + supports_function_calling = True + supports_reasoning = False + supports_response_schema = False + supported_params = frozenset() + kwargs = {} + + async def acall(self, messages, **kwargs): + return [ + { + "text": None, + "tool_calls": [ + { + "id": "call_add", + "type": "function", + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + } + ], + } + ] + + adapter = dspy.TwoStepAdapter(dspy.utils.DummyLM([]), use_native_function_calling=True) + + result = await adapter.acall(NativeToolLM(), {}, ToolSignature, [], {"tools": [dspy.Tool(add)]}) + + assert result[0]["tool_calls"].tool_calls[0].id == "call_add" + + def test_two_step_adapter_parse(): class ComplexSignature(dspy.Signature): input_text: str = dspy.InputField() diff --git a/tests/adapters/test_xml_adapter.py b/tests/adapters/test_xml_adapter.py index ded864ad44..f227f33261 100644 --- a/tests/adapters/test_xml_adapter.py +++ b/tests/adapters/test_xml_adapter.py @@ -8,6 +8,7 @@ import dspy from dspy.adapters.chat_adapter import FieldInfoWithName from dspy.adapters.xml_adapter import XMLAdapter +from tests.adapters.conftest import format_messages_and_lm_kwargs def test_xml_adapter_format_and_parse_basic(): @@ -302,6 +303,452 @@ class QA(dspy.Signature): assert messages[1]["content"] == expected_user +def test_xml_adapter_format_exact_messages_for_simple_signature(): + class StringSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), + StringSignature, + demos=[], + inputs={"question": "why did a chicken cross the kitchen?"}, + ) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str): +All interactions will be structured in the following way, with the appropriate values filled in. + + +{question} + + + +{answer} + +In adhering to this structure, your objective is:\x20 + Given the fields `question`, produce the fields `answer`.""", + }, + { + "role": "user", + "content": """ +why did a chicken cross the kitchen? + + +Respond with the corresponding output fields wrapped in XML tags ``.""", + }, + ] + + +def test_xml_adapter_format_exact_messages_for_two_input_signature(): + class StringSignature(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.InputField() + judgement: str = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), + StringSignature, + demos=[], + inputs={"question": "why did a chicken cross the kitchen?", "answer": "To get to the other side!"}, + ) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str):\x20 +2. `answer` (str): +Your output fields are: +1. `judgement` (str): +All interactions will be structured in the following way, with the appropriate values filled in. + + +{question} + + + +{answer} + + + +{judgement} + +In adhering to this structure, your objective is:\x20 + Given the fields `question`, `answer`, produce the fields `judgement`.""", + }, + { + "role": "user", + "content": """ +why did a chicken cross the kitchen? + + + +To get to the other side! + + +Respond with the corresponding output fields wrapped in XML tags ``.""", + }, + ] + + +def test_xml_adapter_format_exact_messages_with_demo_and_typed_output(): + class MultiAnswer(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + score: float = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), + MultiAnswer, + demos=[{"question": "Q1", "answer": "A1", "score": 0.9}], + inputs={"question": "Q2"}, + ) + + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + assert messages == [ + { + "role": "system", + "content": """Your input fields are: +1. `question` (str): +Your output fields are: +1. `answer` (str):\x20 +2. `score` (float): +All interactions will be structured in the following way, with the appropriate values filled in. + + +{question} + + + +{answer} + + + +{score} # note: the value you produce must be a single float value + +In adhering to this structure, your objective is:\x20 + Given the fields `question`, produce the fields `answer`, `score`.""", + }, + {"role": "user", "content": """ +Q1 +"""}, + { + "role": "assistant", + "content": """ +A1 + + + +0.9 +""", + }, + { + "role": "user", + "content": """ +Q2 + + +Respond with the corresponding output fields wrapped in XML tags ``, then ``.""", + }, + ] + + +def test_xml_adapter_format_exact_messages_with_history_demo_pydantic_tools_and_image(): + def search(query: str, k: int = 3) -> str: + """Search for documents.""" + return query + + class Location(pydantic.BaseModel): + city: str + country: str + + class Profile(pydantic.BaseModel): + name: str + location: Location + interests: list[str] + + class AnswerCard(pydantic.BaseModel): + answer: str + sources: list[str] + + class RichRenderingSignature(dspy.Signature): + """Answer using all supplied context.""" + + history: dspy.History = dspy.InputField() + image: dspy.Image = dspy.InputField() + tools: list[dspy.Tool] = dspy.InputField() + profile: Profile = dspy.InputField() + question: str = dspy.InputField() + answer: AnswerCard = dspy.OutputField() + + tool = dspy.Tool(search) + demo_profile = Profile( + name="Ada", + location=Location(city="London", country="UK"), + interests=["math", "machines"], + ) + current_profile = Profile( + name="Grace", + location=Location(city="Arlington", country="USA"), + interests=["compilers", "navy"], + ) + history = dspy.History( + messages=[ + { + "profile": demo_profile, + "question": "Who is Ada?", + "answer": AnswerCard(answer="Ada is a mathematician.", sources=["memory"]), + } + ] + ) + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), + RichRenderingSignature, + demos=[ + { + "image": dspy.Image("https://example.com/demo.png"), + "tools": [tool], + "profile": demo_profile, + "question": "What should we mention?", + "answer": AnswerCard(answer="Mention analytical engines.", sources=["demo"]), + } + ], + inputs={ + "history": history, + "image": dspy.Image("https://example.com/current.png"), + "tools": [tool], + "profile": current_profile, + "question": "What should the answer include?", + }, + ) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `image` (Image): \n' + '2. `tools` (list[Tool]): \n' + '3. `profile` (Profile): \n' + '4. `question` (str):\n' + 'Your output fields are:\n' + '1. `answer` (AnswerCard):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '\n' + '{image}\n' + '\n' + '\n' + '\n' + '{tools}\n' + '\n' + '\n' + '\n' + '{profile}\n' + '\n' + '\n' + '\n' + '{question}\n' + '\n' + '\n' + '\n' + '{answer} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "properties": {"answer": {"type": "string", "title": "Answer"}, ' + '"sources": {"type": "array", "items": {"type": "string"}, "title": "Sources"}}, ' + '"required": ["answer", "sources"], "title": "AnswerCard"}\n' + '\n' + 'In adhering to this structure, your objective is: \n' + ' Answer using all supplied context.'}, + {"role": "user", + "content": [{"type": "text", "text": "\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/demo.png"}}, + {"type": "text", + "text": '\n' + '\n' + '\n' + '\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '\n' + '\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, ' + '"interests": ["math", "machines"]}\n' + '\n' + '\n' + '\n' + 'What should we mention?\n' + ''}]}, + {"role": "assistant", + "content": '\n{"answer": "Mention analytical engines.", "sources": ["demo"]}\n'}, + {"role": "user", + "content": '\n' + '{"name": "Ada", "location": {"city": "London", "country": "UK"}, "interests": ' + '["math", "machines"]}\n' + '\n' + '\n' + '\n' + 'Who is Ada?\n' + ''}, + {"role": "assistant", + "content": '\n{"answer": "Ada is a mathematician.", "sources": ["memory"]}\n'}, + {"role": "user", + "content": [{"type": "text", "text": "\n"}, + {"type": "image_url", "image_url": {"url": "https://example.com/current.png"}}, + {"type": "text", + "text": '\n' + '\n' + '\n' + '\n' + '["search, whose description is Search for documents.. It ' + "takes arguments {'query': {'type': 'string'}, 'k': {'type': 'integer', " + '\'default\': 3}}."]\n' + '\n' + '\n' + '\n' + '{"name": "Grace", "location": {"city": "Arlington", "country": "USA"}, ' + '"interests": ["compilers", "navy"]}\n' + '\n' + '\n' + '\n' + 'What should the answer include?\n' + '\n' + '\n' + 'Respond with the corresponding output fields wrapped in XML tags ' + '``.'}]}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + +def test_xml_adapter_format_exact_messages_with_nested_pydantic_output(): + class XmlAddress(pydantic.BaseModel): + city: str + country: str + + class XmlSummary(pydantic.BaseModel): + title: str + address: XmlAddress + + class PydanticSignature(dspy.Signature): + question: str = dspy.InputField() + summary: XmlSummary = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), PydanticSignature, [], {"question": "Summarize"}) + + expected_messages = [{"role": "system", + "content": 'Your input fields are:\n' + '1. `question` (str):\n' + 'Your output fields are:\n' + '1. `summary` (XmlSummary):\n' + 'All interactions will be structured in the following way, with the appropriate ' + 'values filled in.\n' + '\n' + '\n' + '{question}\n' + '\n' + '\n' + '\n' + '{summary} # note: the value you produce must adhere to the JSON schema: ' + '{"type": "object", "$defs": {"XmlAddress": {"type": "object", "properties": {"city": ' + '{"type": "string", "title": "City"}, "country": {"type": "string", "title": ' + '"Country"}}, "required": ["city", "country"], "title": "XmlAddress"}}, "properties": ' + '{"address": {"$ref": "#/$defs/XmlAddress"}, "title": {"type": "string", "title": ' + '"Title"}}, "required": ["title", "address"], "title": "XmlSummary"}\n' + '\n' + 'In adhering to this structure, your objective is: \n' + ' Given the fields `question`, produce the fields `summary`.'}, + {"role": "user", + "content": "\n" + "Summarize\n" + "\n" + "\n" + "Respond with the corresponding output fields wrapped in XML tags ``."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + +def test_xml_adapter_format_exact_messages_with_incomplete_demo(): + class IncompleteDemoSignature(dspy.Signature): + question: str = dspy.InputField() + context: str = dspy.InputField() + answer: str = dspy.OutputField() + score: float = dspy.OutputField() + + messages, lm_kwargs = format_messages_and_lm_kwargs(dspy.XMLAdapter(), + IncompleteDemoSignature, + [{"question": "Q1", "answer": "A1"}], + {"question": "Q2", "context": "C2"}, + ) + + expected_messages = [{"role": "system", + "content": "Your input fields are:\n" + "1. `question` (str): \n" + "2. `context` (str):\n" + "Your output fields are:\n" + "1. `answer` (str): \n" + "2. `score` (float):\n" + "All interactions will be structured in the following way, with the appropriate " + "values filled in.\n" + "\n" + "\n" + "{question}\n" + "\n" + "\n" + "\n" + "{context}\n" + "\n" + "\n" + "\n" + "{answer}\n" + "\n" + "\n" + "\n" + "{score} # note: the value you produce must be a single float value\n" + "\n" + "In adhering to this structure, your objective is: \n" + " Given the fields `question`, `context`, produce the fields `answer`, " + "`score`."}, + {"role": "user", + "content": "This is an example of the task, though some input or output fields are not " + "supplied.\n" + "\n" + "\n" + "Q1\n" + ""}, + {"role": "assistant", + "content": "\n" + "A1\n" + "\n" + "\n" + "\n" + "Not supplied for this particular example. \n" + ""}, + {"role": "user", + "content": "\n" + "Q2\n" + "\n" + "\n" + "\n" + "C2\n" + "\n" + "\n" + "Respond with the corresponding output fields wrapped in XML tags ``, then " + "``."}] + assert messages == expected_messages + expected_lm_kwargs = {} + assert lm_kwargs == expected_lm_kwargs + + def test_format_system_message(): class MySignature(dspy.Signature): """Answer the question with multiple answers and scores""" @@ -316,7 +763,7 @@ class MySignature(dspy.Signature): expected_system_message = """Your input fields are: 1. `question` (str): Your output fields are: -1. `answers` (list[str]): +1. `answers` (list[str]):\x20 2. `scores` (list[float]): All interactions will be structured in the following way, with the appropriate values filled in. @@ -331,6 +778,6 @@ class MySignature(dspy.Signature): {scores} # note: the value you produce must adhere to the JSON schema: {"type": "array", "items": {"type": "number"}} -In adhering to this structure, your objective is: +In adhering to this structure, your objective is:\x20 Answer the question with multiple answers and scores""" assert system_message == expected_system_message diff --git a/tests/conftest.py b/tests/conftest.py index 81680d970e..aabbe43faa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,8 @@ import copy import os +from collections.abc import Iterator +from pathlib import Path +from typing import Any import pytest @@ -8,16 +11,31 @@ SKIP_DEFAULT_FLAGS = ["reliability", "extra", "llm_call", "deno"] -@pytest.fixture(autouse=True) -def clear_settings(): - """Ensures that the settings are cleared after each test.""" +def _close_cache(cache: Any) -> None: + disk_cache = getattr(cache, "disk_cache", None) + if hasattr(disk_cache, "close"): + disk_cache.close() - yield +@pytest.fixture(autouse=True) +def clear_settings(tmp_path: Path) -> Iterator[None]: + """Ensure each test gets fresh DSPy settings and an isolated cache.""" import dspy - from dspy.dsp.utils.settings import DEFAULT_CONFIG - dspy.configure(**copy.deepcopy(DEFAULT_CONFIG), inherit_config=False) + original_cache = dspy.cache + dspy.configure_cache(disk_cache_dir=tmp_path / ".dspy_cache") + try: + yield + finally: + from dspy.dsp.utils.settings import DEFAULT_CONFIG + + try: + dspy.configure(**copy.deepcopy(DEFAULT_CONFIG), inherit_config=False) + finally: + try: + _close_cache(dspy.cache) + finally: + dspy.cache = original_cache @pytest.fixture diff --git a/tests/core/test_types.py b/tests/core/test_types.py new file mode 100644 index 0000000000..73a5ca3661 --- /dev/null +++ b/tests/core/test_types.py @@ -0,0 +1,316 @@ +import pydantic +import pytest + +from dspy.core.types import ( + LMAudioPart, + LMConfig, + LMDocumentPart, + LMHistoryEntry, + LMMessage, + LMOutput, + LMOutputBuilder, + LMPromptCacheConfig, + LMReasoningConfig, + LMRequest, + LMRequestPatch, + LMResponse, + LMStreamDeltaEvent, + LMTextDelta, + LMThinkingDelta, + LMThinkingPart, + LMToolCallDelta, + LMToolCallPart, + LMToolChoice, + LMToolResultPart, + LMUsage, + LMVideoPart, + User, +) + + +def _history_entry(message: LMMessage) -> LMHistoryEntry: + return LMHistoryEntry( + request=LMRequest(model="model", messages=[message]), + response=LMResponse.from_text("ok"), + timestamp="timestamp", + uuid="uuid", + ) + + +def test_message_content_and_tool_calls_normalize_for_dspy_history_surface(): + message = LMMessage( + role="assistant", + content="Use search.", + tool_calls=[ + { + "id": "call_1", + "function": {"name": "search", "arguments": '{"query": "dspy"}'}, + } + ], + ) + + assert message.text == "Use search." + assert [part for part in message.parts if isinstance(part, LMToolCallPart)] == [ + LMToolCallPart(id="call_1", name="search", args={"query": "dspy"}) + ] + assert _history_entry(message).messages == [ + { + "role": "assistant", + "content": "Use search.", + "tool_calls": [ + { + "type": "function", + "function": {"name": "search", "arguments": '{"query": "dspy"}'}, + "id": "call_1", + } + ], + } + ] + + +def test_tool_result_content_none_normalizes_to_empty_parts(): + assert LMToolResultPart(content=None).content == [] + + message = LMMessage(role="tool", content=None, tool_call_id="call_1", name="search") + result = message.parts[0] + + assert isinstance(result, LMToolResultPart) + assert result.call_id == "call_1" + assert result.name == "search" + assert result.content == [] + assert _history_entry(message).messages == [ + {"role": "tool", "content": "", "tool_call_id": "call_1", "name": "search"} + ] + + +def test_audio_content_accepts_url_and_history_preserves_url(): + message = LMMessage( + role="user", + content=[ + { + "type": "input_audio", + "input_audio": {"url": "https://example.com/audio.wav", "format": "wav"}, + } + ], + ) + + audio = message.parts[0] + assert isinstance(audio, LMAudioPart) + assert audio.url == "https://example.com/audio.wav" + assert _history_entry(message).messages[0]["content"][0]["input_audio"] == { + "format": "wav", + "url": "https://example.com/audio.wav", + } + + +def test_audio_content_defaults_null_format_to_wav(): + message = LMMessage( + role="user", + content=[ + { + "type": "input_audio", + "input_audio": {"url": "https://example.com/audio.wav", "format": None}, + } + ], + ) + + audio = message.parts[0] + + assert isinstance(audio, LMAudioPart) + assert audio.media_type == "audio/wav" + + +def test_image_content_requires_mapping_with_url(): + with pytest.raises(TypeError, match="Image content block"): + LMMessage(role="user", content=[{"type": "image_url", "image_url": "https://example.com/image.png"}]) + + with pytest.raises(ValueError, match="requires url"): + LMMessage(role="user", content=[{"type": "image_url", "image_url": {}}]) + + +def test_video_data_round_trips_through_history_messages(): + message = User(LMVideoPart(data="YWJj", media_type="video/mp4")) + content = _history_entry(message).messages[0]["content"][0] + round_tripped = LMMessage(role="user", content=[content]).parts[0] + + assert content == { + "type": "video", + "video": {"media_type": "video/mp4", "data": "data:video/mp4;base64,YWJj"}, + } + assert isinstance(round_tripped, LMVideoPart) + assert round_tripped.data == "YWJj" + assert round_tripped.media_type == "video/mp4" + + +def test_document_source_url_stays_url_and_round_trips_through_history_messages(): + message = LMMessage( + role="user", + content=[ + { + "type": "document", + "source": "https://example.com/report.pdf", + "title": "Report", + } + ], + ) + document = message.parts[0] + content = _history_entry(message).messages[0]["content"][0] + round_tripped = LMMessage(role="user", content=[content]).parts[0] + + assert isinstance(document, LMDocumentPart) + assert document.url == "https://example.com/report.pdf" + assert document.data is None + assert isinstance(round_tripped, LMDocumentPart) + assert round_tripped.url == "https://example.com/report.pdf" + assert round_tripped.data is None + + +def test_config_extensions_flatten_when_converted_to_legacy_kwargs(): + config = LMConfig.from_kwargs(temperature=0.2, provider_flag=True) + patch = LMRequestPatch(config=config) + request = LMRequest.from_call(model="model", prompt="hi", temperature=0.2, provider_flag=True) + entry = LMHistoryEntry( + request=request, + response=LMResponse.from_text("ok"), + timestamp="timestamp", + uuid="uuid", + ) + + assert patch.as_lm_kwargs() == {"provider_flag": True, "temperature": 0.2} + assert entry.kwargs == {"provider_flag": True, "temperature": 0.2} + + +def test_lm_kwargs_aliases_normalize_for_existing_dspy_lm_callers(): + config = LMConfig.from_kwargs( + reasoning={"effort": "low", "summary": "auto"}, + reasoning_effort="high", + tool_choice={"mode": "auto"}, + parallel_tool_calls=False, + cache=True, + rollout_id=7, + prompt_cache=True, + prompt_cache_key="prompt-cache", + provider_flag=True, + ) + + assert config.reasoning.effort == "high" + assert config.reasoning.summary == "auto" + assert config.tool_choice.mode == "auto" + assert config.tool_choice.parallel is False + assert config.cache.enabled is True + assert config.cache.rollout_id == 7 + assert config.prompt_cache.enabled is True + assert config.prompt_cache.key == "prompt-cache" + assert config.extensions == {"provider_flag": True} + + +def test_nested_config_aliases_remain_supported_for_existing_interfaces(): + reasoning = LMReasoningConfig.from_value({"reasoning_effort": "high"}) + tool_choice = LMToolChoice.from_value({"parallel_tool_calls": False}) + prompt_cache = LMPromptCacheConfig.from_value({"prompt_cache_key": "prompt-cache"}) + + assert reasoning.effort == "high" + assert tool_choice.mode == "auto" + assert tool_choice.parallel is False + assert prompt_cache.key == "prompt-cache" + + +def test_usage_normalizes_existing_user_visible_token_aliases(): + provider_usage = LMUsage(prompt_tokens=1, completion_tokens=2) + canonical_usage = LMUsage(input_tokens=1, output_tokens=2) + + assert provider_usage.input_tokens == 1 + assert provider_usage.output_tokens == 2 + assert provider_usage.total_tokens == 3 + assert canonical_usage.prompt_tokens == 1 + assert canonical_usage.completion_tokens == 2 + assert canonical_usage.total_tokens == 3 + + +def test_default_config_does_not_serialize_empty_stop_sequences(): + request = LMRequest.from_call(model="model", prompt="hi") + entry = LMHistoryEntry( + request=request, + response=LMResponse.from_text("ok"), + timestamp="timestamp", + uuid="uuid", + ) + + assert request.config.stop is None + assert entry.kwargs == {} + + +def test_history_entry_exposes_existing_dict_style_properties(): + message = User("hi") + request = LMRequest.from_call(model="model", messages=[message], temperature=0.2) + response = LMResponse.from_text("ok", model="response-model", usage={"input_tokens": 1}, cost=0.5) + entry = LMHistoryEntry(request=request, response=response, timestamp="timestamp", uuid="uuid") + + assert entry.model == "model" + assert entry.prompt == "hi" + assert entry.messages == [{"role": "user", "content": "hi"}] + assert entry.outputs == ["ok"] + assert entry.usage["input_tokens"] == 1 + assert entry.cost == 0.5 + assert entry.kwargs == {"temperature": 0.2} + assert entry.response_model == "response-model" + assert entry["messages"] == [{"role": "user", "content": "hi"}] + assert entry["kwargs"] == {"temperature": 0.2} + + +def test_response_rejects_empty_outputs(): + with pytest.raises(pydantic.ValidationError): + LMResponse(model="model", outputs=[]) + + +def test_output_to_value_preserves_redacted_thinking_part(): + thinking = LMThinkingPart(text="hidden", redacted=True) + output = LMOutput(parts=[thinking]) + + assert output.to_value() == [thinking] + + +def test_stream_event_indices_must_be_non_negative(): + with pytest.raises(pydantic.ValidationError): + LMStreamDeltaEvent(output_index=-1, part_index=0, delta=LMTextDelta(text="x")) + + with pytest.raises(pydantic.ValidationError): + LMStreamDeltaEvent(output_index=0, part_index=-1, delta=LMTextDelta(text="x")) + + +def test_stream_builder_rejects_sparse_output_indices(): + builder = LMOutputBuilder() + builder.apply(LMStreamDeltaEvent(output_index=2, part_index=0, delta=LMTextDelta(text="third"))) + + with pytest.raises(ValueError, match="output indices"): + builder.to_response() + + +def test_stream_builder_rejects_sparse_part_indices(): + builder = LMOutputBuilder() + builder.apply(LMStreamDeltaEvent(output_index=0, part_index=1, delta=LMTextDelta(text="second"))) + + with pytest.raises(ValueError, match="part indices"): + builder.to_response() + + +def test_stream_builder_rejects_delta_type_changes(): + builder = LMOutputBuilder() + builder.apply(LMStreamDeltaEvent(output_index=0, part_index=0, delta=LMTextDelta(text="text"))) + + with pytest.raises(ValueError, match="thinking delta"): + builder.apply(LMStreamDeltaEvent(output_index=0, part_index=0, delta=LMThinkingDelta(text="thought"))) + + +def test_stream_builder_rejects_incomplete_tool_call_arguments(): + builder = LMOutputBuilder() + builder.apply( + LMStreamDeltaEvent( + output_index=0, + part_index=0, + delta=LMToolCallDelta(id="call_1", name="search", args_delta='{"query": '), + ) + ) + + with pytest.raises(ValueError, match="tool-call arguments"): + builder.to_response() diff --git a/tests/predict/test_reactv2.py b/tests/predict/test_reactv2.py new file mode 100644 index 0000000000..f074b3d57c --- /dev/null +++ b/tests/predict/test_reactv2.py @@ -0,0 +1,179 @@ +import dspy +from dspy.predict.reactv2 import ReActV2, ToolExecutionResult, _build_submit_tool +from dspy.utils.dummies import DummyLM + + +def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + +def test_submit_tool_returns_dict(): + signature = dspy.Signature("question -> answer") + submit = _build_submit_tool(signature) + + assert submit(answer="42") == {"answer": "42"} + + +def test_basic_forward_with_submit_records_history_messages(): + lm = DummyLM( + [ + {"next_thought": "I should add.", "tool_calls": [{"name": "add", "args": {"a": 1, "b": 2}}]}, + {"next_thought": "I have the answer.", "tool_calls": [{"name": "submit", "args": {"answer": "3"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="What is 1+2?") + + assert result.answer == "3" + assert len(result.history.messages) == 2 + assert result.history.messages[0]["question"] == "What is 1+2?" + assert result.history.messages[0]["next_thought"] == "I should add." + assert result.history.messages[0]["tool_call_results"].tool_call_results[0].value == 3 + assert result.history.messages[-1]["answer"] == "3" + second_call_user_messages = [message["content"] for message in lm.history[1]["messages"] if message["role"] == "user"] + assert any("tool_call_results" in content for content in second_call_user_messages) + assert all("None" not in content for content in second_call_user_messages) + assert "What is 1+2?" in second_call_user_messages[0] + assert "What is 1+2?" not in second_call_user_messages[-1] + + +def test_forward_with_native_tool_calling_renders_tool_results_as_tool_messages(): + class NativeToolLoopLM(dspy.BaseLM): + def __init__(self): + super().__init__(model="openai/gpt-5-nano", cache=False) + self.calls = [] + + @property + def supported_params(self): + return frozenset() + + @property + def supports_function_calling(self): + return True + + @property + def supports_reasoning(self): + return False + + @property + def supports_response_schema(self): + return False + + def __call__(self, messages, **kwargs): + self.calls.append({"messages": messages, "kwargs": kwargs}) + if len(self.calls) == 1: + return [ + { + "text": "[[ ## next_thought ## ]]\nI should add.\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "function": {"name": "add", "arguments": '{"a": 1, "b": 2}'}, + "id": "call_add", + "type": "function", + } + ], + } + ] + return [ + { + "text": "[[ ## next_thought ## ]]\nI can answer.\n\n[[ ## completed ## ]]", + "tool_calls": [ + { + "function": {"name": "submit", "arguments": '{"answer": "3"}'}, + "id": "call_submit", + "type": "function", + } + ], + } + ] + + lm = NativeToolLoopLM() + with dspy.context(lm=lm, adapter=dspy.ChatAdapter(use_native_function_calling=True)): + result = ReActV2("question -> answer", tools=[add])(question="What is 1+2?") + + assert result.answer == "3" + assert result.history.messages[0]["tool_call_results"].tool_call_results[0].call_id == "call_add" + assert any(message["role"] == "tool" and message["tool_call_id"] == "call_add" for message in lm.calls[1]["messages"]) + assert lm.calls[0]["kwargs"]["tools"][0]["function"]["name"] == "add" + + +def test_react_signature_keeps_optional_tool_call_results_input(): + react = ReActV2("question -> answer", tools=[add]) + + assert react.react.signature.input_fields["question"].default is None + field = react.react.signature.input_fields["tool_call_results"] + assert field.default is None + assert field.annotation == dspy.ToolCallResults | None + + +def test_forward_with_existing_history_does_not_append_empty_input_event(): + lm = DummyLM( + [ + {"next_thought": "I have the answer.", "tool_calls": [{"name": "submit", "args": {"answer": "3"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + history = dspy.History(messages=[{"question": "What is 1+2?"}]) + + result = react(history=history) + + assert result.answer == "3" + assert {} not in result.history.messages + assert result.history.messages[0] == {"question": "What is 1+2?"} + + +def test_unknown_tool_returns_error_observation(): + lm = DummyLM( + [ + {"next_thought": "Call fake.", "tool_calls": [{"name": "nonexistent", "args": {}}]}, + {"next_thought": "Now submit.", "tool_calls": [{"name": "submit", "args": {"answer": "ok"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="test") + + assert result.answer == "ok" + tool_results = [ + result + for message in result.history.messages + for result in getattr(message.get("tool_call_results"), "tool_call_results", []) + ] + assert any(result.is_error and "Unknown tool" in str(result.value) for result in tool_results) + + +def test_append_tool_turn_records_observation_ids(): + history = dspy.History(messages=[]) + tool_calls = dspy.ToolCalls.from_dict_list([{"name": "add", "args": {"a": 1, "b": 2}, "id": "call_add"}]) + + ReActV2._append_tool_turn( + history, + next_thought="add", + tool_calls=tool_calls, + tool_results=[ToolExecutionResult(value=3)], + ) + + tool_result = history.messages[0]["tool_call_results"].tool_call_results[0] + assert tool_result.call_id == "call_add" + assert tool_result.name == "add" + + +def test_forced_submit_runs_when_loop_returns_no_tool_calls(): + lm = DummyLM( + [ + {"next_thought": "No call.", "tool_calls": []}, + {"next_thought": "Force submit.", "tool_calls": [{"name": "submit", "args": {"answer": "done"}}]}, + ] + ) + dspy.configure(lm=lm) + react = ReActV2("question -> answer", tools=[add]) + + result = react(question="test") + + assert result.answer == "done" + assert result.termination_reason == "forced_submit" diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py index 315aa5b1c7..6cdbf4363f 100644 --- a/tests/streaming/test_streaming.py +++ b/tests/streaming/test_streaming.py @@ -1059,16 +1059,15 @@ def is_streamable(cls) -> bool: return True @classmethod - def adapt_to_native_lm_feature(cls, signature, field_name, lm, lm_kwargs): - return signature.delete(field_name) + def default_parse_output(cls, ctx): + return CustomType(message=ctx.adapter.lm_parts_to_text(ctx.parts)) @classmethod def parse_stream_chunk(cls, chunk): - return CustomType(message=chunk.choices[0].delta.content) - - @classmethod - def parse_lm_response(cls, response: dict) -> "CustomType": - return CustomType(message=response.split("\n\n")[0]) + content = chunk.choices[0].delta.content + if content in {"Hello", "World"}: + return CustomType(message=content) + return None class CustomSignature(dspy.Signature): question: str = dspy.InputField() @@ -1082,6 +1081,9 @@ class CustomSignature(dspy.Signature): ) async def stream(*args, **kwargs): + yield ModelResponseStream( + model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ## answer ## ]]\n\n"))] + ) yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="Hello"))]) yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="World"))]) yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="\n\n"))]) @@ -1091,9 +1093,7 @@ async def stream(*args, **kwargs): yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content=" ]]"))]) with mock.patch("litellm.acompletion", side_effect=stream): - with dspy.context( - lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter(native_response_types=[CustomType]) - ): + with dspy.context(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter()): output = program(question="why did a chicken cross the kitchen?") all_chunks = [] async for value in output: @@ -1104,6 +1104,7 @@ async def stream(*args, **kwargs): assert value.answer.message == "HelloWorld" assert all(isinstance(chunk.chunk, CustomType) for chunk in all_chunks) + assert [chunk.chunk.message for chunk in all_chunks] == ["Hello", "World"] @pytest.mark.anyio