11"""Fit the numeric decision parameters of every Predict in a program against the metric.
22
3- A decision backend's probabilities often track the label and lean. Fixing the lean needs no new
4- wording. Each `Predict` holds per-output parameters in `fields` that reinterpret the backend's
3+ Each `Predict` holds per-output parameters in `fields` that reinterpret the backend's
54probabilities without changing the request, so cached answers are reused and each candidate
65setting is one pass of plain Python over the training set. The search runs directly against the
76metric, whatever shape the program's outputs have.
109- `threshold`, a Boolean output's cut point on P(True).
1110- `cuts`, a Score's boundaries on the mean level index. They pick the returned `.level` and leave
1211 `.value` alone.
13- - `weights`, a Choice's multiplier per option label, from 0.1 to 10, applied to the probabilities before the
12+ - `weights`, a Choice's multiplier per option label, applied to the probabilities before the
1413 option is picked. An option picked too often gets a multiplier below 1.
1514
15+ The candidate settings come from the training set. One pass records the probabilities the output
16+ is decoded from on every call. A setting anywhere between two neighbouring observed values makes
17+ the same decisions, so the candidates are the midpoints of those gaps: between P(True) values for
18+ a threshold, between mean level indexes for a cut, and between the points where an option's pick
19+ flips for a weight. A setting stays unless a candidate scores strictly better, and among equal
20+ scores the candidate in the widest gap wins.
21+
1622On a generative LM, a native `bool` or `Literal` output returns its value without probabilities
1723unless it has an entry in `fields`. Calibration adds that entry, which asks the LM for
1824probabilities, and keeps it only when the fitted setting scores strictly better than the native
2834"""
2935
3036import copy
37+ import itertools
3138import math
3239from typing import Any , Callable
3340
3441import dspy
35- from dspy .adapters .decision import resolve_adapter
42+ from dspy .adapters .decision import record_evidence , resolve_adapter
3643from dspy .adapters .types .decision import Choice , Noul , Score , decision_type
3744from dspy .predict .predict import Predict
3845from dspy .utils .parallelizer import ParallelExecutor
3946
40- THRESHOLDS = [round (0.05 * i , 2 ) for i in range (1 , 20 )]
41- MULTIPLIERS = [0.1 , 0.15 , 0.25 , 0.35 , 0.5 , 0.7 , 1.0 , 1.4 , 2.0 , 2.8 , 4.0 , 6.5 , 10.0 ] # one Choice option's multiplier
42- SCORE_STEPS = 10 # a Score's level range is searched in this many steps; each cut is then refined by half a step
47+ MAX_CANDIDATES = 40 # candidate settings per parameter; more distinct values are thinned to quantiles
48+ WEIGHT_RANGE = (1e-3 , 1e3 ) # the smallest and largest Choice multiplier tried
4349
4450
4551def decision_outputs (predict : Predict ) -> dict [str , type ]:
@@ -151,19 +157,16 @@ def score() -> float:
151157 _restore (predict , field , original )
152158 report .append ({"predictor" : name , "field" : field , "skipped" : "the metric does not read this output" })
153159 continue
160+ evidence = _observe (program , predict , field , trainset , metric , num_threads )
154161 row = {"predictor" : name , "field" : field }
155162 if issubclass (kind , Noul ):
156- row .update (_fit_threshold (predict , field , score ))
157- elif issubclass (kind , Choice ):
158- base = score ()
159- best , best_score = _fit_multipliers (predict , field , kind , score , base )
160- row .update (parameter = "weights" , value = best , train_score = round (best_score , 4 ))
161- row ["train_score_at_start" ] = round (base , 4 )
163+ row .update (_fit_threshold (predict , field , evidence , score ))
162164 else :
163165 base = score ()
164- best , best_score = _fit_cuts (predict , field , kind , score , base )
165- row .update (parameter = "cuts" , value = best , train_score = round (best_score , 4 ))
166- row ["train_score_at_start" ] = round (base , 4 )
166+ fit = _fit_weights if issubclass (kind , Choice ) else _fit_cuts
167+ best , best_score , observed = fit (predict , field , kind , evidence , score , base )
168+ row .update (parameter = "weights" if fit is _fit_weights else "cuts" , value = best )
169+ row .update (train_score = round (best_score , 4 ), train_score_at_start = round (base , 4 ), observed = observed )
167170 if promoted :
168171 row ["train_score_native" ] = round (native , 4 )
169172 if row ["train_score" ] <= row ["train_score_native" ]:
@@ -189,26 +192,78 @@ def _restore(predict: Predict, field: str, original: dict | None) -> None:
189192 predict .fields [field ] = original
190193
191194
192- def _fit_threshold (predict : Predict , field : str , score : Callable ) -> dict [str , Any ]:
193- """The threshold that scores best. The starting threshold stays unless a grid value scores strictly better."""
195+ def _observe (program , predict : Predict , field : str , trainset : list , metric : Callable , num_threads ) -> list [dict ]:
196+ """The evidence the output is decoded from on each training call, from one pass at the current setting."""
197+ with record_evidence () as log :
198+ scores (program , trainset , metric , num_threads )
199+ return [evidence for caller , name , evidence in log if caller is predict and name == field ]
200+
201+
202+ def _tidy (x : float , lo : float , hi : float ) -> float :
203+ """The shortest decimal strictly between `lo` and `hi`, rounded from `x`."""
204+ for digits in range (1 , 12 ):
205+ rounded = round (x , digits )
206+ if lo < rounded < hi :
207+ return rounded
208+ return x
209+
210+
211+ def _gaps (values : list [float ], lo : float , hi : float , geometric : bool = False ) -> list [tuple [float , float ]]:
212+ """Candidate settings and the width of the gap each sits in.
213+
214+ The candidates are the midpoints between neighbouring distinct values, with `lo` and `hi`
215+ closing the range. A setting anywhere inside one gap makes the same decisions as its midpoint,
216+ so these cover every outcome. More than MAX_CANDIDATES gaps are thinned to evenly spaced
217+ quantiles of the values. `geometric` takes midpoints and widths on a log scale.
218+ """
219+ inner = sorted ({v for v in values if lo < v < hi })
220+ if len (inner ) > MAX_CANDIDATES - 1 :
221+ step = (len (inner ) - 1 ) / (MAX_CANDIDATES - 2 )
222+ inner = sorted ({inner [round (i * step )] for i in range (MAX_CANDIDATES - 1 )})
223+ points = [lo , * inner , hi ]
224+ gaps = []
225+ for a , b in itertools .pairwise (points ):
226+ middle , width = (math .sqrt (a * b ), math .log (b / a )) if geometric else ((a + b ) / 2 , b - a )
227+ gaps .append ((_tidy (middle , a , b ), width ))
228+ return gaps
229+
230+
231+ def _summary (values : list [float ]) -> dict [str , Any ]:
232+ distinct = sorted (set (values ))
233+ if not distinct :
234+ return {"calls" : 0 }
235+ return {
236+ "calls" : len (values ),
237+ "distinct" : len (distinct ),
238+ "min" : round (distinct [0 ], 4 ),
239+ "max" : round (distinct [- 1 ], 4 ),
240+ }
241+
242+
243+ def _fit_threshold (predict : Predict , field : str , evidence : list [dict ], score : Callable ) -> dict [str , Any ]:
244+ """The threshold that scores best, between the observed probabilities.
245+
246+ The starting threshold stays unless a candidate scores strictly better. Among equal scores, the
247+ candidate in the widest gap wins, since it leaves the most room on either side.
248+ """
194249 config = predict .fields [field ]
195250 start = config ["threshold" ]
196251 base = score ()
197- found = {}
198- for t in THRESHOLDS :
252+ probabilities = [e ["noul" ] for e in evidence ]
253+ tried = []
254+ for t , width in _gaps (probabilities , 0.0 , 1.0 ):
199255 config ["threshold" ] = t
200- found [t ] = base if t == start else score ()
201- best = max (found , key = lambda t : (found [t ], - abs (t - 0.5 )))
202- if found [best ] <= base :
203- best = start
204- found [start ] = base
256+ tried .append ((score (), width , - abs (t - start ), t ))
257+ best_score , _ , _ , best = max (tried ) if tried else (base , 0 , 0 , start )
258+ if best_score <= base :
259+ best , best_score = start , base
205260 config ["threshold" ] = best
206261 return {
207262 "parameter" : "threshold" ,
208263 "value" : best ,
209- "train_score" : round (found [ best ] , 4 ),
264+ "train_score" : round (best_score , 4 ),
210265 "train_score_at_start" : round (base , 4 ),
211- "train_score_at_default " : round ( found [ 0.5 ], 4 ) ,
266+ "observed " : { ** _summary ( probabilities ), "candidates" : len ( tried )} ,
212267 }
213268
214269
@@ -239,63 +294,72 @@ def _ignored(program, predict: Predict, field: str, kind: type, trainset: list,
239294 return all (values == seen [0 ] for values in seen [1 :])
240295
241296
242- def _fit_multipliers (predict : Predict , field : str , kind : type , score : Callable , base : float ) -> tuple [dict , float ]:
297+ def _fit_weights (
298+ predict : Predict , field : str , kind : type , evidence : list [dict ], score : Callable , base : float
299+ ) -> tuple [dict , float , dict ]:
243300 """The Choice multipliers that score best on the training set.
244301
245- Each option's multiplier moves in turn over a log-scale grid while the others hold. Ties go to
246- the multipliers nearest 1.
302+ Each option's multiplier moves in turn while the others hold. An option's pick on a call flips
303+ where its weighted probability meets the strongest rival's, so the candidates sit between those
304+ flip points. A candidate replaces the current multipliers only when it scores strictly better.
247305 """
248306 labels = [str (value ) for value , _ in kind .options ]
249307 config = predict .fields [field ]
250- start = {label : config ["weights" ].get (label , 1.0 ) for label in labels }
251-
252- def distance (weights ):
253- return sum (abs (math .log (w )) for w in weights .values ())
254-
255- best , best_score = start , base
308+ best = {label : config ["weights" ].get (label , 1.0 ) for label in labels }
309+ best_score , candidates = base , 0
256310 for label in labels :
257- for w in MULTIPLIERS :
311+ flips = []
312+ for e in evidence :
313+ p = e ["probabilities" ]
314+ rival = max (best [other ] * p [other ] for other in labels if other != label )
315+ if p [label ] > 0 and rival > 0 :
316+ flips .append (rival / p [label ])
317+ low , high = WEIGHT_RANGE
318+ if flips :
319+ low , high = max (low , min (flips ) / 4 ), min (high , max (flips ) * 4 )
320+ tried = []
321+ for w , width in _gaps (flips , low , high , geometric = True ):
258322 if w == best [label ]:
259323 continue
260324 candidate = {** best , label : w }
261325 config ["weights" ] = candidate
262- sc = score ()
263- if sc > best_score or (sc == best_score and distance (candidate ) < distance (best )):
326+ tried .append ((score (), width , candidate ))
327+ candidates += len (tried )
328+ if tried :
329+ sc , _ , candidate = max (tried , key = lambda t : t [:2 ])
330+ if sc > best_score :
264331 best , best_score = candidate , sc
265332 config ["weights" ] = best
266- return best , best_score
333+ return best , best_score , { "calls" : len ( evidence ), "candidates" : candidates }
267334
268335
269- def _fit_cuts (predict : Predict , field : str , kind : type , score : Callable , base : float ) -> tuple [list [float ], float ]:
336+ def _fit_cuts (
337+ predict : Predict , field : str , kind : type , evidence : list [dict ], score : Callable , base : float
338+ ) -> tuple [list [float ], float , dict ]:
270339 """The Score cuts that score best on the training set.
271340
272- Each cut moves in turn over a grid strictly inside the level range, then by half a step either
273- way, and stays strictly between its neighbours. Ties go to the cuts nearest the defaults,
274- halfway between levels, so a metric that ignores `.level` keeps them. A three-level Score
275- costs about two dozen passes.
341+ A call's level depends only on its mean level index. Each cut moves in turn between the
342+ observed means that lie between its neighbours, and stays strictly between them. A candidate
343+ replaces the current cuts only when it scores strictly better.
276344 """
277345 config = predict .fields [field ]
278346 top = len (kind .options ) - 1
279- default = [i + 0.5 for i in range (top )]
280- step = top / SCORE_STEPS
281-
282- def distance (cuts ):
283- return sum (abs (c - d ) for c , d in zip (cuts , default , strict = True ))
284-
285- best , best_score = list (config ["cuts" ]), base
347+ means = [sum (i * p for i , p in e ["probabilities" ].items ()) / sum (e ["probabilities" ].values ()) for e in evidence ]
348+ best , best_score , candidates = list (config ["cuts" ]), base , 0
286349 for i in range (len (best )):
287- coarse = [round (step * j , 6 ) for j in range (1 , SCORE_STEPS )]
288- for stage in ("coarse" , "fine" ):
289- grid = coarse if stage == "coarse" else [round (best [i ] - step / 2 , 6 ), round (best [i ] + step / 2 , 6 )]
290- for c in grid :
291- below = best [i - 1 ] if i else 0
292- above = best [i + 1 ] if i + 1 < len (best ) else top
293- if not below < c < above or c == best [i ]:
294- continue
295- candidate = [* best [:i ], c , * best [i + 1 :]]
296- config ["cuts" ] = candidate
297- sc = score ()
298- if sc > best_score or (sc == best_score and distance (candidate ) < distance (best )):
299- best , best_score = candidate , sc
350+ below = best [i - 1 ] if i else 0
351+ above = best [i + 1 ] if i + 1 < len (best ) else top
352+ tried = []
353+ for c , width in _gaps (means , below , above ):
354+ if c == best [i ]:
355+ continue
356+ candidate = [* best [:i ], c , * best [i + 1 :]]
357+ config ["cuts" ] = candidate
358+ tried .append ((score (), width , candidate ))
359+ candidates += len (tried )
360+ if tried :
361+ sc , _ , candidate = max (tried , key = lambda t : t [:2 ])
362+ if sc > best_score :
363+ best , best_score = candidate , sc
300364 config ["cuts" ] = best
301- return best , best_score
365+ return best , best_score , { ** _summary ( means ), "candidates" : candidates }
0 commit comments