Skip to content

Commit f35f57f

Browse files
dbreunigclaude
andcommitted
feat(gepa): add code_proposer hook for custom dspy.Flex code proposals
dspy.GEPA accepts a custom instruction_proposer, but dspy.Flex code components were hardcoded to the built-in propose_code. Add the symmetric hook: a code_proposer parameter on dspy.GEPA, routed through DspyAdapter.propose_new_texts under the reflection LM context, with the built-in proposer as the default. Suppress the 'skips code components' warning when a custom code proposer handles them. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 15c62d4 commit f35f57f

3 files changed

Lines changed: 182 additions & 8 deletions

File tree

‎dspy/teleprompt/gepa/gepa.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,13 @@
1111

1212
from dspy.clients.lm import LM
1313
from dspy.primitives import Example, Module, Prediction
14-
from dspy.teleprompt.gepa.gepa_utils import DspyAdapter, DSPyTrace, PredictorFeedbackFn, ScoreWithFeedback
14+
from dspy.teleprompt.gepa.gepa_utils import (
15+
CodeProposalFn,
16+
DspyAdapter,
17+
DSPyTrace,
18+
PredictorFeedbackFn,
19+
ScoreWithFeedback,
20+
)
1521
from dspy.teleprompt.teleprompt import Teleprompter
1622
from dspy.utils.annotation import experimental
1723

@@ -268,6 +274,21 @@ def metric(
268274
Note: When both instruction_proposer and reflection_lm are set, the instruction_proposer is called
269275
in the reflection_lm context. However, reflection_lm is optional when using a custom instruction_proposer.
270276
Custom instruction proposers can invoke their own LLMs if needed.
277+
code_proposer: Optional custom proposer for `dspy.Flex` code components, implementing the
278+
`CodeProposalFn` protocol from `dspy.teleprompt.gepa.gepa_utils`.
279+
**Default: None** - Uses the built-in code proposer, which rewrites each Flex
280+
submodule's full `module_src` from the failing examples and feedback.
281+
282+
A custom code proposer is called once per reflection round with the code components to
283+
update, the current candidate, the reflective dataset, and per-component task
284+
descriptions (the rendered Flex signature) and context blurbs (available tools and
285+
style notes). It must return a complete replacement `dspy.Module` subclass source per
286+
component. Like instruction_proposer, it is invoked inside the reflection_lm context
287+
when one is set, and may invoke its own LM otherwise.
288+
289+
Only Flex components are routed to it; regular predictors keep going to
290+
instruction_proposer (or the default instruction proposer). This parameter has no
291+
effect on programs without `dspy.Flex` submodules.
271292
component_selector: Custom component selector implementing the [ReflectionComponentSelector](https://github.com/gepa-ai/gepa/blob/main/src/gepa/proposer/reflective_mutation/base.py) protocol,
272293
or a string specifying a built-in selector strategy. Controls which components (predictors) are selected
273294
for optimization at each iteration. Defaults to 'round_robin' strategy which cycles through components
@@ -365,6 +386,7 @@ def __init__(
365386
skip_perfect_score: bool = True,
366387
add_format_failure_as_feedback: bool = False,
367388
instruction_proposer: "ProposalFn | None" = None,
389+
code_proposer: "CodeProposalFn | None" = None,
368390
component_selector: "ReflectionComponentSelector | str" = "round_robin",
369391
# Merge-based configuration
370392
use_merge: bool = True,
@@ -448,6 +470,7 @@ def __init__(
448470
self.seed = seed
449471

450472
self.custom_instruction_proposer = instruction_proposer
473+
self.custom_code_proposer = code_proposer
451474
self.component_selector = component_selector
452475
self.gepa_kwargs = gepa_kwargs or {}
453476

@@ -590,6 +613,7 @@ def feedback_fn(
590613
rng=rng,
591614
reflection_lm=self.reflection_lm,
592615
custom_instruction_proposer=self.custom_instruction_proposer,
616+
custom_code_proposer=self.custom_code_proposer,
593617
warn_on_score_mismatch=self.warn_on_score_mismatch,
594618
reflection_minibatch_size=self.reflection_minibatch_size,
595619
)

‎dspy/teleprompt/gepa/gepa_utils.py‎

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,25 @@ def log(self, x: str):
5252
"""
5353

5454

55+
class CodeProposalFn(Protocol):
56+
"""Custom proposer for dspy.Flex code components.
57+
58+
The code-component counterpart of gepa's ProposalFn: called with the code components to
59+
update plus the per-component task descriptions (rendered Flex signatures) and context
60+
blurbs (available tools and style notes), under the reflection LM's context. Returns a
61+
full replacement module source (one dspy.Module subclass) per component.
62+
"""
63+
64+
def __call__(
65+
self,
66+
candidate: dict[str, str],
67+
reflective_dataset: dict[str, list[dict[str, Any]]],
68+
components_to_update: list[str],
69+
task_descriptions: dict[str, str],
70+
context_blurbs: dict[str, str],
71+
) -> dict[str, str]: ...
72+
73+
5574
class ScoreWithFeedback(Prediction):
5675
score: float
5776
feedback: str
@@ -95,6 +114,7 @@ def __init__(
95114
rng: random.Random | None = None,
96115
reflection_lm=None,
97116
custom_instruction_proposer: "ProposalFn | None" = None,
117+
custom_code_proposer: "CodeProposalFn | None" = None,
98118
warn_on_score_mismatch: bool = True,
99119
reflection_minibatch_size: int | None = None,
100120
):
@@ -107,6 +127,7 @@ def __init__(
107127
self.rng = rng or random.Random(0)
108128
self.reflection_lm = reflection_lm
109129
self.custom_instruction_proposer = custom_instruction_proposer
130+
self.custom_code_proposer = custom_code_proposer
110131
self.warn_on_score_mismatch = warn_on_score_mismatch
111132
self.reflection_minibatch_size = reflection_minibatch_size
112133
self._warned_custom_proposer_skips_code = False
@@ -124,23 +145,37 @@ def propose_new_texts(
124145
) -> dict[str, str]:
125146
reflection_lm = self.reflection_lm or dspy.settings.lm
126147

127-
# dspy.Flex code components are rewritten by the code proposer.
148+
# dspy.Flex code components are rewritten by the code proposer. A custom code proposer
149+
# overrides the built-in one, mirroring how custom_instruction_proposer overrides the
150+
# default instruction proposer below.
128151
results: dict[str, str] = {}
129152
code_keys = [c for c in components_to_update if c in self._flex_paths]
130153
if code_keys:
131-
results.update(
132-
propose_code(
133-
code_keys, candidate, reflective_dataset,
134-
self._flex_task_descriptions, self._flex_context_blurbs, reflection_lm,
154+
if self.custom_code_proposer:
155+
with dspy.context(lm=reflection_lm):
156+
results.update(
157+
self.custom_code_proposer(
158+
candidate=candidate,
159+
reflective_dataset=reflective_dataset,
160+
components_to_update=code_keys,
161+
task_descriptions=self._flex_task_descriptions,
162+
context_blurbs=self._flex_context_blurbs,
163+
)
164+
)
165+
else:
166+
results.update(
167+
propose_code(
168+
code_keys, candidate, reflective_dataset,
169+
self._flex_task_descriptions, self._flex_context_blurbs, reflection_lm,
170+
)
135171
)
136-
)
137172
components_to_update = [c for c in components_to_update if c not in self._flex_paths]
138173
if not components_to_update:
139174
return results
140175

141176
# A custom proposer overrides the default *instruction* proposer only.
142177
if self.custom_instruction_proposer:
143-
if code_keys and not self._warned_custom_proposer_skips_code:
178+
if code_keys and not self.custom_code_proposer and not self._warned_custom_proposer_skips_code:
144179
logger.warning(
145180
"A custom instruction_proposer is set, but %d dspy.Flex code component(s) are "
146181
"being optimized. The custom proposer handles instruction components only; the "

‎tests/flex/test_flex_gepa.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,102 @@ def forward(self, *args, **kwargs):
407407
assert out == {"self": original}
408408

409409

410+
# --- adapter: a custom code proposer overrides the built-in one ---------------
411+
412+
413+
def test_custom_code_proposer_overrides_builtin_and_receives_contract() -> None:
414+
captured = {}
415+
416+
def code_proposer(*, candidate, reflective_dataset, components_to_update, task_descriptions, context_blurbs):
417+
captured.update(
418+
candidate=candidate,
419+
reflective_dataset=reflective_dataset,
420+
components_to_update=components_to_update,
421+
task_descriptions=task_descriptions,
422+
context_blurbs=context_blurbs,
423+
)
424+
return dict.fromkeys(components_to_update, SIMPLE_MODULE)
425+
426+
student = dspy.Flex(Echo)
427+
adapter = DspyAdapter(
428+
student_module=student,
429+
metric_fn=_metric,
430+
feedback_map={},
431+
reflection_lm=DummyLM([]), # empty: any call through the built-in proposer would fail
432+
custom_code_proposer=code_proposer,
433+
)
434+
candidate = {"self": student.module_src}
435+
reflective = {"self": [{"Inputs": {"q": "x"}, "Generated Outputs": "wrong", "Feedback": "bad"}]}
436+
437+
out = adapter.propose_new_texts(candidate, reflective, ["self"])
438+
439+
assert out == {"self": SIMPLE_MODULE}
440+
assert captured["components_to_update"] == ["self"]
441+
assert captured["candidate"] == candidate
442+
assert captured["reflective_dataset"] == reflective
443+
assert "self" in captured["task_descriptions"]
444+
assert "self" in captured["context_blurbs"]
445+
446+
447+
def test_custom_code_proposer_runs_in_reflection_lm_context() -> None:
448+
seen = {}
449+
reflection = DummyLM([])
450+
451+
def code_proposer(*, candidate, reflective_dataset, components_to_update, task_descriptions, context_blurbs):
452+
seen["lm"] = dspy.settings.lm
453+
return dict.fromkeys(components_to_update, SIMPLE_MODULE)
454+
455+
student = dspy.Flex(Echo)
456+
adapter = DspyAdapter(
457+
student_module=student,
458+
metric_fn=_metric,
459+
feedback_map={},
460+
reflection_lm=reflection,
461+
custom_code_proposer=code_proposer,
462+
)
463+
adapter.propose_new_texts({"self": student.module_src}, {"self": []}, ["self"])
464+
assert seen["lm"] is reflection
465+
466+
467+
def test_no_skip_warning_when_custom_code_proposer_handles_code(caplog) -> None:
468+
"""The 'custom instruction_proposer skips code components' warning exists because those
469+
components would otherwise fall through to the built-in code proposer silently. With a
470+
custom code proposer installed, nothing is skipped, so the warning must not fire."""
471+
472+
class Prog(dspy.Module):
473+
def __init__(self):
474+
super().__init__()
475+
self.flex = dspy.Flex(Echo)
476+
self.sibling = dspy.Predict("x -> y")
477+
478+
def forward(self, **kwargs):
479+
return self.flex(**kwargs)
480+
481+
def instruction_proposer(*, candidate, reflective_dataset, components_to_update):
482+
return dict.fromkeys(components_to_update, "new instruction")
483+
484+
def code_proposer(*, candidate, reflective_dataset, components_to_update, task_descriptions, context_blurbs):
485+
return dict.fromkeys(components_to_update, SIMPLE_MODULE)
486+
487+
prog = Prog()
488+
adapter = DspyAdapter(
489+
student_module=prog,
490+
metric_fn=_metric,
491+
feedback_map={},
492+
reflection_lm=DummyLM([]),
493+
custom_instruction_proposer=instruction_proposer,
494+
custom_code_proposer=code_proposer,
495+
)
496+
candidate = {"flex": prog.flex.module_src, "sibling": "old instruction"}
497+
reflective = {"flex": [], "sibling": []}
498+
499+
with caplog.at_level("WARNING"):
500+
out = adapter.propose_new_texts(candidate, reflective, ["flex", "sibling"])
501+
502+
assert out == {"flex": SIMPLE_MODULE, "sibling": "new instruction"}
503+
assert not any("code component" in r.message for r in caplog.records)
504+
505+
410506
# --- GEPA.compile: seed mixes code + instruction components ------------------
411507

412508

@@ -448,6 +544,25 @@ def fake_optimize(**kwargs):
448544
assert not any(k.startswith("flex.") for k in seed)
449545

450546

547+
def test_gepa_compile_threads_code_proposer_to_adapter(monkeypatch) -> None:
548+
captured = {}
549+
550+
def fake_optimize(**kwargs):
551+
captured["adapter"] = kwargs["adapter"]
552+
return _FakeResult(best_candidate=kwargs["seed_candidate"])
553+
554+
monkeypatch.setattr("gepa.optimize", fake_optimize)
555+
556+
def code_proposer(*, candidate, reflective_dataset, components_to_update, task_descriptions, context_blurbs):
557+
return dict.fromkeys(components_to_update, SIMPLE_MODULE)
558+
559+
optimizer = dspy.GEPA(metric=_metric, reflection_lm=DummyLM([]), max_metric_calls=10, code_proposer=code_proposer)
560+
ex = dspy.Example(q="x", a="y").with_inputs("q")
561+
optimizer.compile(dspy.Flex(Echo), trainset=[ex], valset=[ex])
562+
563+
assert captured["adapter"].custom_code_proposer is code_proposer
564+
565+
451566
# --- DspyGEPAResult.to_dict: candidates keep their code components -----------
452567

453568

0 commit comments

Comments
 (0)