@@ -12,6 +12,8 @@ import {
1212 Eval ,
1313 EvalScorer ,
1414 runEvaluator ,
15+ _internalPrepareEvaluatorScore ,
16+ type OneOrMoreScores ,
1517} from "./framework" ;
1618import {
1719 _exportsForTestingOnly ,
@@ -2180,6 +2182,101 @@ test("scorer-only evaluator populates scores field", async () => {
21802182 expect ( result . results [ 0 ] . classifications ) . toBeUndefined ( ) ;
21812183} ) ;
21822184
2185+ test ( "single score objects use the same names as numeric returns" , async ( ) => {
2186+ const result = await Eval (
2187+ "test-nameless-scores" ,
2188+ {
2189+ data : [ { input : "hello" } ] ,
2190+ task : ( input ) => input ,
2191+ scores : [
2192+ function accuracy ( ) {
2193+ return { score : 0.8 } ;
2194+ } ,
2195+ async function relevance ( ) {
2196+ return { score : 1 , metadata : { reason : "relevant" } } ;
2197+ } ,
2198+ ( ) => ( { score : 0 } ) ,
2199+ ( ) => ( { name : "explicit" , score : 0.5 } ) ,
2200+ ] ,
2201+ } ,
2202+ { noSendLogs : true , returnResults : true } ,
2203+ ) ;
2204+
2205+ expect ( result . results [ 0 ] . scores ) . toEqual ( {
2206+ accuracy : 0.8 ,
2207+ relevance : 1 ,
2208+ scorer_2 : 0 ,
2209+ explicit : 0.5 ,
2210+ } ) ;
2211+ } ) ;
2212+
2213+ describe ( "scorer result normalization" , ( ) => {
2214+ test . each ( [ 0 , 0.8 , null ] ) (
2215+ "defaults the name and preserves fields for %s" ,
2216+ ( score ) => {
2217+ const value = Object . freeze ( { score, metadata : { reason : "test" } } ) ;
2218+ expect ( _internalPrepareEvaluatorScore ( value , "accuracy" ) ) . toEqual ( {
2219+ results : [ { ...value , name : "accuracy" } ] ,
2220+ output : { score } ,
2221+ metadata : value . metadata ,
2222+ scores : { accuracy : score } ,
2223+ } ) ;
2224+ expect ( value ) . not . toHaveProperty ( "name" ) ;
2225+ } ,
2226+ ) ;
2227+
2228+ test . each ( [ "explicit" , "" ] ) ( "preserves an explicit name %j" , ( name ) => {
2229+ expect (
2230+ _internalPrepareEvaluatorScore ( { name, score : 1 } , "fallback" ) . scores ,
2231+ ) . toEqual ( { [ name ] : 1 } ) ;
2232+ } ) ;
2233+
2234+ test ( "keeps named arrays and numeric returns working" , ( ) => {
2235+ expect ( _internalPrepareEvaluatorScore ( 0.8 , "accuracy" ) . scores ) . toEqual ( {
2236+ accuracy : 0.8 ,
2237+ } ) ;
2238+ expect ( _internalPrepareEvaluatorScore ( null , "accuracy" ) ) . toEqual ( {
2239+ results : null ,
2240+ } ) ;
2241+ expect (
2242+ _internalPrepareEvaluatorScore (
2243+ [
2244+ { name : "accuracy" , score : 0.8 } ,
2245+ { name : "relevance" , score : 1 } ,
2246+ ] ,
2247+ "fallback" ,
2248+ ) . scores ,
2249+ ) . toEqual ( { accuracy : 0.8 , relevance : 1 } ) ;
2250+ } ) ;
2251+
2252+ test ( "requires names in arrays at typecheck and runtime" , ( ) => {
2253+ // @ts -expect-error A single-element array still requires a named score.
2254+ const unnamed : OneOrMoreScores = [ { score : 1 } ] ;
2255+ const mixed : OneOrMoreScores = [
2256+ { name : "accuracy" , score : 1 } ,
2257+ // @ts -expect-error Every entry in a mixed array must have a name.
2258+ { score : 0.5 } ,
2259+ ] ;
2260+ for ( const value of [ unnamed , mixed ] ) {
2261+ expect ( ( ) => _internalPrepareEvaluatorScore ( value , "fallback" ) ) . toThrow (
2262+ "each score must have a name" ,
2263+ ) ;
2264+ }
2265+ } ) ;
2266+
2267+ test ( "rejects duplicate array names" , ( ) => {
2268+ expect ( ( ) =>
2269+ _internalPrepareEvaluatorScore (
2270+ [
2271+ { name : "accuracy" , score : 1 } ,
2272+ { name : "accuracy" , score : 0.5 } ,
2273+ ] ,
2274+ "fallback" ,
2275+ ) ,
2276+ ) . toThrow ( "Duplicate score name 'accuracy'" ) ;
2277+ } ) ;
2278+ } ) ;
2279+
21832280test ( "multiple classifiers returning the same name append items correctly" , async ( ) => {
21842281 const result = await Eval (
21852282 "test-classifier-append" ,
0 commit comments