Skip to content

Commit ef0da25

Browse files
authored
1 parent 99971a9 commit ef0da25

8 files changed

Lines changed: 149 additions & 9 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"braintrust": minor
3+
---
4+
5+
feat: Allow nameless scorer results

‎e2e/scenarios/durable-eval-webhook/scenario.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ async function main() {
6868
return items.map((item) => ({
6969
id: item.id,
7070
score: {
71-
name: "batch_exact",
7271
score: item.output === item.expected ? 1 : 0,
7372
metadata: { method: "batch-provider" },
7473
},
@@ -90,7 +89,6 @@ async function main() {
9089
scores: [
9190
function exact({ output, expected }) {
9291
return {
93-
name: "exact",
9492
score: output === expected ? 1 : 0,
9593
metadata: { method: "shared-eval-runtime" },
9694
};

‎js/src/framework.test.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import {
1212
Eval,
1313
EvalScorer,
1414
runEvaluator,
15+
_internalPrepareEvaluatorScore,
16+
type OneOrMoreScores,
1517
} from "./framework";
1618
import {
1719
_exportsForTestingOnly,
@@ -2180,6 +2182,101 @@ test("scorer-only evaluator populates scores field", async () => {
21802182
expect(result.results[0].classifications).toBeUndefined();
21812183
});
21822184

2185+
test("single score objects use the same names as numeric returns", async () => {
2186+
const result = await Eval(
2187+
"test-nameless-scores",
2188+
{
2189+
data: [{ input: "hello" }],
2190+
task: (input) => input,
2191+
scores: [
2192+
function accuracy() {
2193+
return { score: 0.8 };
2194+
},
2195+
async function relevance() {
2196+
return { score: 1, metadata: { reason: "relevant" } };
2197+
},
2198+
() => ({ score: 0 }),
2199+
() => ({ name: "explicit", score: 0.5 }),
2200+
],
2201+
},
2202+
{ noSendLogs: true, returnResults: true },
2203+
);
2204+
2205+
expect(result.results[0].scores).toEqual({
2206+
accuracy: 0.8,
2207+
relevance: 1,
2208+
scorer_2: 0,
2209+
explicit: 0.5,
2210+
});
2211+
});
2212+
2213+
describe("scorer result normalization", () => {
2214+
test.each([0, 0.8, null])(
2215+
"defaults the name and preserves fields for %s",
2216+
(score) => {
2217+
const value = Object.freeze({ score, metadata: { reason: "test" } });
2218+
expect(_internalPrepareEvaluatorScore(value, "accuracy")).toEqual({
2219+
results: [{ ...value, name: "accuracy" }],
2220+
output: { score },
2221+
metadata: value.metadata,
2222+
scores: { accuracy: score },
2223+
});
2224+
expect(value).not.toHaveProperty("name");
2225+
},
2226+
);
2227+
2228+
test.each(["explicit", ""])("preserves an explicit name %j", (name) => {
2229+
expect(
2230+
_internalPrepareEvaluatorScore({ name, score: 1 }, "fallback").scores,
2231+
).toEqual({ [name]: 1 });
2232+
});
2233+
2234+
test("keeps named arrays and numeric returns working", () => {
2235+
expect(_internalPrepareEvaluatorScore(0.8, "accuracy").scores).toEqual({
2236+
accuracy: 0.8,
2237+
});
2238+
expect(_internalPrepareEvaluatorScore(null, "accuracy")).toEqual({
2239+
results: null,
2240+
});
2241+
expect(
2242+
_internalPrepareEvaluatorScore(
2243+
[
2244+
{ name: "accuracy", score: 0.8 },
2245+
{ name: "relevance", score: 1 },
2246+
],
2247+
"fallback",
2248+
).scores,
2249+
).toEqual({ accuracy: 0.8, relevance: 1 });
2250+
});
2251+
2252+
test("requires names in arrays at typecheck and runtime", () => {
2253+
// @ts-expect-error A single-element array still requires a named score.
2254+
const unnamed: OneOrMoreScores = [{ score: 1 }];
2255+
const mixed: OneOrMoreScores = [
2256+
{ name: "accuracy", score: 1 },
2257+
// @ts-expect-error Every entry in a mixed array must have a name.
2258+
{ score: 0.5 },
2259+
];
2260+
for (const value of [unnamed, mixed]) {
2261+
expect(() => _internalPrepareEvaluatorScore(value, "fallback")).toThrow(
2262+
"each score must have a name",
2263+
);
2264+
}
2265+
});
2266+
2267+
test("rejects duplicate array names", () => {
2268+
expect(() =>
2269+
_internalPrepareEvaluatorScore(
2270+
[
2271+
{ name: "accuracy", score: 1 },
2272+
{ name: "accuracy", score: 0.5 },
2273+
],
2274+
"fallback",
2275+
),
2276+
).toThrow("Duplicate score name 'accuracy'");
2277+
});
2278+
});
2279+
21832280
test("multiple classifiers returning the same name append items correctly", async () => {
21842281
const result = await Eval(
21852282
"test-classifier-append",

‎js/src/framework.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
SpanTypeAttribute,
88
spanObjectTypeV3ToTypedString,
99
} from "../util/index";
10+
import type { SingleScore } from "../util/score";
1011
import {
1112
type GitMetadataSettingsType as GitMetadataSettings,
1213
ObjectReference as ObjectReferenceSchema,
@@ -177,7 +178,7 @@ export type EvalScorerArgs<
177178
trace?: Trace;
178179
};
179180

180-
export type OneOrMoreScores = Score | number | null | Array<Score>;
181+
export type OneOrMoreScores = SingleScore | number | null | Array<Score>;
181182

182183
export type EvalScorer<
183184
Input,
@@ -1045,19 +1046,29 @@ export function _internalPrepareEvaluatorScore(
10451046
} {
10461047
if (scoreValue === null) return { results: null };
10471048
if (Array.isArray(scoreValue)) {
1049+
const names = new Set<string>();
10481050
for (const score of scoreValue) {
10491051
if (!(typeof score === "object" && !isEmpty(score))) {
10501052
throw new Error(
10511053
`When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(score)}`,
10521054
);
10531055
}
1056+
if (typeof score.name !== "string") {
1057+
throw new Error(
1058+
`When returning an array of scores, each score must have a name. Got: ${JSON.stringify(score)}`,
1059+
);
1060+
}
1061+
if (names.has(score.name)) {
1062+
throw new Error(`Duplicate score name '${score.name}' in score array`);
1063+
}
1064+
names.add(score.name);
10541065
}
10551066
}
10561067
let results: Score[];
10571068
if (Array.isArray(scoreValue)) {
10581069
results = scoreValue;
10591070
} else if (typeof scoreValue === "object" && !isEmpty(scoreValue)) {
1060-
results = [scoreValue];
1071+
results = [{ ...scoreValue, name: scoreValue.name ?? name }];
10611072
} else {
10621073
results = [{ name, score: scoreValue }];
10631074
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { afterEach, expect, test, vi } from "vitest";
2+
import { NOOP_SPAN } from "../../logger";
3+
import { runScorers } from "./scorers";
4+
import type { ScorerFunction } from "./types";
5+
6+
afterEach(() => vi.restoreAllMocks());
7+
8+
test.each<ScorerFunction>([
9+
() => ({ score: 0.8, metadata: { reason: "test" } }),
10+
async () => ({ score: 0.8, metadata: { reason: "test" } }),
11+
])("test runner scorers accept nameless objects", async (scorer) => {
12+
const log = vi.spyOn(NOOP_SPAN, "log");
13+
await runScorers({
14+
scorers: [scorer],
15+
input: "hello",
16+
output: "hello",
17+
expected: "hello",
18+
metadata: undefined,
19+
span: NOOP_SPAN,
20+
});
21+
expect(log).toHaveBeenCalledWith({
22+
scores: { score: 0.8 },
23+
metadata: { reason: "test" },
24+
});
25+
});

‎js/src/wrappers/shared/scorers.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type { Span } from "../../logger";
22
import type { Score } from "../../../util/score";
33
import type { ScorerFunction } from "./types";
4+
import type { OneOrMoreScores } from "../../framework";
45

56
export async function runScorers(args: {
67
scorers: ScorerFunction[];
@@ -60,7 +61,7 @@ function isScore(val: object): val is Score {
6061
return "name" in val && "score" in val;
6162
}
6263

63-
function normalizeScores(result: unknown): Score[] {
64+
function normalizeScores(result: OneOrMoreScores): Score[] {
6465
if (result === null || result === undefined) {
6566
return [];
6667
}
@@ -76,8 +77,8 @@ function normalizeScores(result: unknown): Score[] {
7677
);
7778
}
7879

79-
if (typeof result === "object" && result !== null && isScore(result)) {
80-
return [result];
80+
if (typeof result === "object" && result !== null && "score" in result) {
81+
return [{ ...result, name: result.name ?? "score" }];
8182
}
8283

8384
return [];

‎js/src/wrappers/shared/types.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
import type { Score } from "../../../util/score";
1+
import type { OneOrMoreScores } from "../../framework";
22

33
// Scorer function type
44
export type ScorerFunction<Output = unknown> = (args: {
55
output: Output;
66
expected?: unknown;
77
input?: unknown;
88
metadata?: Record<string, unknown>;
9-
}) => Score | Promise<Score> | number | null | Array<Score>;
9+
}) => OneOrMoreScores | Promise<OneOrMoreScores>;
1010

1111
// Progress event types for real-time test reporting
1212
export type ProgressEvent =

‎js/util/score.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ export interface Score {
4747
error?: unknown;
4848
}
4949

50+
// A single result can use the scorer's name; array entries must stay named.
51+
export type SingleScore = Omit<Score, "name"> & { name?: string };
52+
5053
export type ScorerArgs<Output, Extra> = {
5154
output: Output;
5255
expected?: Output;

0 commit comments

Comments
 (0)