diff --git a/dspy/_vendor/lm15-provenance.txt b/dspy/_vendor/lm15-provenance.txt index fe0e7f89ba..338d87095c 100644 --- a/dspy/_vendor/lm15-provenance.txt +++ b/dspy/_vendor/lm15-provenance.txt @@ -1,5 +1,5 @@ source=https://github.com/cmpnd-ai/lm15-python.git version=1.0.0rc3 -commit=55fce2e61a04ed3193acfeb8797b7c58d1b55398 +commit=b0bffc13bc4a12e115e4c2e92fbba40ca8f3450c contract=b721ce4f03877d72a91431b3a5ce653ffc0b3e38 -split=63b6bb319f38ad1d31060138da54cb8ac8718711 +split=6a745c84e83a58aa4ed1237132d940f78ce65c35 diff --git a/dspy/_vendor/lm15/providers/openai_chat.py b/dspy/_vendor/lm15/providers/openai_chat.py index f5a6eab2d6..35f32e77df 100644 --- a/dspy/_vendor/lm15/providers/openai_chat.py +++ b/dspy/_vendor/lm15/providers/openai_chat.py @@ -246,7 +246,13 @@ def _response_format_to_chat(format_config: dict[str, Any]) -> dict[str, Any]: _INGEST_GROQ_BUILTIN_INVERSE: dict[str, str] = {wire: name for name, wire in _GROQ_BUILTIN_MAP.items()} -_INGEST_AUDIO_MEDIA_TYPES: dict[str, str] = {"wav": "audio/wav", "mp3": "audio/mpeg"} +# MAP-12 rule 4: OpenAI's server takes wav and mp3, Gemini's any audio type, +# and DSPy writes the MIME subtype (mpeg for .mp3). Each format reads as its +# true media type; a builder with no audio slot raises at send (MAP-10). +_INGEST_AUDIO_MEDIA_TYPES: dict[str, str] = { + "wav": "audio/wav", "mp3": "audio/mpeg", "mpeg": "audio/mpeg", "ogg": "audio/ogg", "opus": "audio/opus", + "flac": "audio/flac", "aac": "audio/aac", "aiff": "audio/aiff", "webm": "audio/webm", +} def _ingest_unsupported(provider: str, what: str, why: str) -> UnsupportedFeatureError: diff --git a/tests/clients/test_lm15_audio_formats.py b/tests/clients/test_lm15_audio_formats.py new file mode 100644 index 0000000000..4f7373a6fb --- /dev/null +++ b/tests/clients/test_lm15_audio_formats.py @@ -0,0 +1,29 @@ +import json + +import dspy +import dspy.clients.execution as execution +from dspy._vendor.lm15.testing import FakeResponse, FakeTransport +from dspy.lm15 import RouterConfig + + +def test_ogg_audio_reaches_gemini_inline_with_its_media_type(monkeypatch): + reply = { + "candidates": [{"content": {"role": "model", "parts": [{"text": "[[ ## text ## ]]\nhej\n\n[[ ## completed ## ]]"}]}, + "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}, + } + transport = FakeTransport([FakeResponse(status=200, body=json.dumps(reply).encode())]) + monkeypatch.setattr(execution, "RouterConfig", lambda **kwargs: RouterConfig( + **{**{k: v for k, v in kwargs.items() if k != "timeouts"}, "api_keys": {"gemini": "fake"}, "transport": transport}, + )) + + class Transcribe(dspy.Signature): + audio: dspy.Audio = dspy.InputField() + text: str = dspy.OutputField() + + with dspy.context(lm=dspy.LM("gemini/gemini-3.8-flash", cache=False), adapter=dspy.ChatAdapter()): + result = dspy.Predict(Transcribe)(audio=dspy.Audio(data="T2dnUw==", audio_format="ogg")) + + assert result.text == "hej" + parts = json.loads(transport.requests[0].body)["contents"][-1]["parts"] + assert {"inlineData": {"mimeType": "audio/ogg", "data": "T2dnUw=="}} in parts