fix(adapters): serialize non-JSON values one at a time in serialize_for_json (#10634)

One value pydantic can't serialize (e.g. a numpy scalar from a pandas row)
made serialize_for_json fall back to str() of the whole value. With
InstructionProposer as GEPA's default, this turned every reflection example
into a Python repr, dropped dspy.Image parts, and ignored custom __str__.
Pass fallback=str so only the offending value is stringified.

Amp-Thread-ID: https://ampcode.com/threads/T-01a12205-bc16-7750-9c23-8d86826ea0ec

Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Isaac Miller <isaac@cmpnd.ai>

Author: 3 people

dspy/adapters/utils.py

@@ -71,11 +71,11 @@ def serialize_for_json(value: Any) -> Any:
     Returns:
         The formatted value, which is serializable as a JSON string.
     """
-    # Attempt to format the value as a JSON-compatible object using pydantic, falling back to
-    # a string representation of the value if that fails (e.g. if the value contains an object
-    # that pydantic doesn't recognize or can't serialize)
+    # Format the value as a JSON-compatible object using pydantic. A nested value pydantic can't
+    # serialize (e.g. a numpy scalar) becomes its string form without affecting its siblings; the
+    # whole value falls back to its string form only if pydantic can't handle it at all.
     try:
-        return TypeAdapter(type(value)).dump_python(value, mode="json")
+        return TypeAdapter(type(value)).dump_python(value, mode="json", fallback=str)
     except Exception:
         return str(value)
 

tests/teleprompt/test_gepa_builtin_instruction_proposer.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 from typing import Any
 
+import numpy as np
 import pytest
 from pydantic import ValidationError
 
@@ -643,12 +644,23 @@ def format(self, signature, demos, inputs):
             seen.append(inputs["examples_with_feedback"])
             return super().format(signature, demos, inputs)
 
+    class Claim:
+        def __str__(self):
+            return "claim #7"
+
     history = tool_history("short result")
     image = dspy.Image(url="https://example.com/image.png")
     repl = REPLHistory().append(code="print(1)", output="1")
     examples = [
         {
-            "Inputs": {"Context": history, "nested": [{"image": image}], "repl": repl},
+            "Inputs": {
+                "Context": history,
+                "nested": [{"image": image}],
+                "repl": repl,
+                # Values pydantic can't serialize, e.g. a numpy scalar read from a pandas row.
+                "amount": np.int64(1200),
+                "claim": Claim(),
+            },
             "Generated Outputs": {"answer": "cat"},
             "Feedback": "ok",
         }
@@ -660,6 +672,9 @@ def format(self, signature, demos, inputs):
     assert seen[0][0]["Inputs"]["repl"] is repl
     content = lm.history[-1]["messages"][-1]["content"]
     assert any(part.get("type") == "image_url" for part in content)
+    text = "".join(part.get("text", "") for part in content)
+    assert '"amount": "1200", "claim": "claim #7"' in text
+    assert '"Feedback": "ok"' in text
 
 
 @pytest.mark.parametrize("builtin", [True, False])