|
| 1 | +"""QC-07 gates: JSONAdapter cutover dual-run (request + parse), BAML legacy |
| 2 | +routing, shared-machinery regression on chat, and JSON parse fuzzing.""" |
| 3 | + |
| 4 | +import json |
| 5 | +import random |
| 6 | + |
| 7 | +import pytest |
| 8 | +from golden.cases import CASES, case_fixture, execute_case |
| 9 | +from golden.generate_fixtures import GOLDEN_DIR, REQUEST_DIR, render_fixture |
| 10 | +from golden.harness import canonical_error, canonicalize |
| 11 | +from golden.parse_cases import PARSE_CASES, parse_case_fixture |
| 12 | + |
| 13 | +import dspy |
| 14 | +from dspy.adapters._engine.formats import resolve_format |
| 15 | +from dspy.adapters._engine.formats.json import JSONFormat |
| 16 | +from dspy.adapters._engine.overrides import resolve_override_verdict |
| 17 | +from dspy.adapters.baml_adapter import BAMLAdapter |
| 18 | +from dspy.adapters.json_adapter import JSONAdapter |
| 19 | + |
| 20 | +JSON_REQUEST_CASE_IDS = sorted(case_id for case_id, case in CASES.items() if case.adapter == "json") |
| 21 | +JSON_PARSE_CASE_IDS = sorted(case_id for case_id, case in PARSE_CASES.items() if case.adapter == "json") |
| 22 | +BAML_REQUEST_CASE_IDS = sorted(case_id for case_id, case in CASES.items() if case.adapter == "baml") |
| 23 | +CHAT_REQUEST_CASE_IDS = sorted(case_id for case_id, case in CASES.items() if case.adapter == "chat") |
| 24 | + |
| 25 | + |
| 26 | +class _ForcedLegacyJSON(JSONAdapter): |
| 27 | + """Pass-through format override routes the whole instance legacy.""" |
| 28 | + |
| 29 | + def format(self, signature, demos, inputs): |
| 30 | + return super().format(signature, demos, inputs) |
| 31 | + |
| 32 | + |
| 33 | +def _forced_legacy_factory(case, recorder): |
| 34 | + return _ForcedLegacyJSON(**case.adapter_kwargs) |
| 35 | + |
| 36 | + |
| 37 | +def test_json_adapter_is_engine_backed_with_json_format(): |
| 38 | + adapter = JSONAdapter() |
| 39 | + assert resolve_override_verdict(adapter).engine_eligible |
| 40 | + assert isinstance(resolve_format(adapter), JSONFormat) |
| 41 | + # ChatAdapter still resolves ChatFormat, not JSONFormat. |
| 42 | + from dspy.adapters._engine.formats.chat import ChatFormat |
| 43 | + from dspy.adapters.chat_adapter import ChatAdapter |
| 44 | + |
| 45 | + assert type(resolve_format(ChatAdapter())) is ChatFormat |
| 46 | + |
| 47 | + |
| 48 | +def test_baml_adapter_routes_legacy_and_stays_unexported(): |
| 49 | + assert not resolve_override_verdict(BAMLAdapter()).engine_eligible |
| 50 | + import dspy.adapters |
| 51 | + |
| 52 | + assert not hasattr(dspy.adapters, "BAMLAdapter") |
| 53 | + |
| 54 | + |
| 55 | +@pytest.mark.parametrize("case_id", JSON_REQUEST_CASE_IDS) |
| 56 | +def test_json_request_dual_run_engine_matches_fixture(case_id): |
| 57 | + actual = json.loads(render_fixture(case_fixture(CASES[case_id]))) |
| 58 | + expected = json.loads((REQUEST_DIR / f"{case_id}.json").read_text(encoding="utf-8")) |
| 59 | + assert actual == expected |
| 60 | + |
| 61 | + |
| 62 | +@pytest.mark.parametrize("case_id", JSON_REQUEST_CASE_IDS) |
| 63 | +def test_json_request_dual_run_legacy_matches_fixture(case_id): |
| 64 | + case = CASES[case_id] |
| 65 | + legacy_expected = canonicalize(execute_case(case, adapter_factory=_forced_legacy_factory)) |
| 66 | + fixture = json.loads((REQUEST_DIR / f"{case_id}.json").read_text(encoding="utf-8")) |
| 67 | + assert legacy_expected == fixture["expected"] |
| 68 | + |
| 69 | + |
| 70 | +@pytest.mark.parametrize("case_id", JSON_PARSE_CASE_IDS) |
| 71 | +def test_json_parse_dual_run_engine_matches_fixture(case_id): |
| 72 | + actual = json.loads(render_fixture(parse_case_fixture(PARSE_CASES[case_id]))) |
| 73 | + expected = json.loads((GOLDEN_DIR / "parse" / f"{case_id}.json").read_text(encoding="utf-8")) |
| 74 | + assert actual == expected |
| 75 | + |
| 76 | + |
| 77 | +@pytest.mark.parametrize("case_id", BAML_REQUEST_CASE_IDS) |
| 78 | +def test_baml_request_corpus_unchanged(case_id): |
| 79 | + """BAML (unmigrated, overriding subclass) must reproduce its fixtures |
| 80 | + through the legacy path it automatically keeps.""" |
| 81 | + actual = json.loads(render_fixture(case_fixture(CASES[case_id]))) |
| 82 | + expected = json.loads((REQUEST_DIR / f"{case_id}.json").read_text(encoding="utf-8")) |
| 83 | + assert actual == expected |
| 84 | + |
| 85 | + |
| 86 | +@pytest.mark.parametrize("case_id", CHAT_REQUEST_CASE_IDS) |
| 87 | +def test_chat_corpus_unchanged_by_shared_machinery(case_id): |
| 88 | + """JSONFormat subclasses ChatFormat: prove the shared machinery did not |
| 89 | + shift a single chat byte.""" |
| 90 | + from golden.generate_fixtures import _python_minor_differs |
| 91 | + |
| 92 | + if getattr(CASES[case_id], "python_sensitive", False) and _python_minor_differs(): |
| 93 | + pytest.skip("python-minor-dependent docstring bytes; the pinned parity CI job is the byte authority") |
| 94 | + actual = json.loads(render_fixture(case_fixture(CASES[case_id]))) |
| 95 | + expected = json.loads((REQUEST_DIR / f"{case_id}.json").read_text(encoding="utf-8")) |
| 96 | + assert actual == expected |
| 97 | + |
| 98 | + |
| 99 | +# --- JSON parse fuzzing --------------------------------------------------------- |
| 100 | + |
| 101 | +SEED = 0x77AB1 |
| 102 | +N_CASES = 100 |
| 103 | + |
| 104 | +WRAPPERS = [ |
| 105 | + lambda text: text, |
| 106 | + lambda text: f"```json\n{text}\n```", |
| 107 | + lambda text: f"Here is the answer:\n{text}\nHope that helps!", |
| 108 | + lambda text: text.replace('"', "'", 3), |
| 109 | + lambda text: text[:-3], |
| 110 | + lambda text: "[" + text + "]", |
| 111 | + lambda text: text + ' {"sneaky": "second object"}', |
| 112 | + lambda text: "not json at all", |
| 113 | + lambda text: "", |
| 114 | +] |
| 115 | + |
| 116 | + |
| 117 | +def _fuzz_case(index): |
| 118 | + rng = random.Random(SEED + index) |
| 119 | + names = rng.sample(["alpha", "beta", "gamma", "delta"], rng.randint(1, 3)) |
| 120 | + fields = {name: (str, dspy.OutputField()) for name in names} |
| 121 | + fields["seed_input"] = (str, dspy.InputField()) |
| 122 | + signature = dspy.make_signature(fields, None, signature_name="FuzzJSONSignature") |
| 123 | + |
| 124 | + payload = { |
| 125 | + name: rng.choice(["value", "multi\nline", "", "unicode «π» 🧪", '"quoted"']) |
| 126 | + for name in names |
| 127 | + if rng.random() < 0.9 |
| 128 | + } |
| 129 | + completion = rng.choice(WRAPPERS)(json.dumps(payload, indent=rng.choice([None, 2]))) |
| 130 | + return signature, completion |
| 131 | + |
| 132 | + |
| 133 | +@pytest.mark.parametrize("index", range(N_CASES)) |
| 134 | +def test_json_parse_fuzz_engine_equals_legacy(index): |
| 135 | + signature, completion = _fuzz_case(index) |
| 136 | + |
| 137 | + def run(adapter): |
| 138 | + try: |
| 139 | + return ("completed", canonicalize(adapter.parse(signature, completion))) |
| 140 | + except Exception as error: |
| 141 | + return (type(error).__name__, canonicalize(canonical_error(error))) |
| 142 | + |
| 143 | + assert run(JSONAdapter()) == run(_ForcedLegacyJSON()), f"json parse fuzz divergence at index {index}" |
0 commit comments