From e2e23057c583e6a9532463ff2b6dc30f2f5b2628 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:15:34 +0000 Subject: [PATCH 1/2] feat: Add decisions prompt support --- e2e/config/pr-comment-scenarios.json | 5 + .../prompt-decisions/scenario.test.ts | 35 +++ e2e/scenarios/prompt-decisions/scenario.ts | 57 ++++ js/src/cli/util/pull.ts | 21 +- js/src/eval-parameters.ts | 2 +- js/src/framework2.test.ts | 28 ++ js/src/framework2.ts | 22 +- js/src/logger.test.ts | 32 +++ js/src/logger.ts | 143 +++++++--- js/src/prompt-schemas.ts | 90 +++++-- js/src/prompt.test.ts | 254 +++++++++++++++++- 11 files changed, 604 insertions(+), 85 deletions(-) create mode 100644 e2e/scenarios/prompt-decisions/scenario.test.ts create mode 100644 e2e/scenarios/prompt-decisions/scenario.ts diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 451aae354..e6316f9cd 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -19,6 +19,11 @@ "label": "Trace primitives and project caching", "metadataScenario": "trace-primitives-basic" }, + { + "scenarioDirName": "prompt-decisions", + "label": "Decisions prompt building", + "metadataScenario": "prompt-decisions" + }, { "scenarioDirName": "ts-exec-loader", "label": "TypeScript loader compatibility", diff --git a/e2e/scenarios/prompt-decisions/scenario.test.ts b/e2e/scenarios/prompt-decisions/scenario.test.ts new file mode 100644 index 000000000..65286da7d --- /dev/null +++ b/e2e/scenarios/prompt-decisions/scenario.test.ts @@ -0,0 +1,35 @@ +import { expect, test } from "vitest"; +import { + prepareScenarioDir, + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { findLatestSpan } from "../../helpers/trace-selectors"; + +const scenarioDir = await prepareScenarioDir({ + scenarioDir: resolveScenarioDir(import.meta.url), +}); + +test("builds decisions prompts from the packaged SDK", async () => { + await withScenarioHarness(async ({ runScenarioDir, testRunEvents }) => { + await runScenarioDir({ scenarioDir }); + const root = findLatestSpan(testRunEvents(), "prompt-decisions"); + expect(root?.output).toEqual({ + model: "decision-model", + input: [{ role: "user", content: "Evaluate hello" }], + questions: [ + { type: "predicate", instructions: "Is hello a greeting?" }, + { + type: "choice", + instructions: "Choose the category", + choices: [{ value: "greeting" }, { value: false }], + }, + { + type: "score", + instructions: "Rate clarity", + levels: [{ label: "clear", description: "Very clear" }], + }, + ], + }); + }); +}); diff --git a/e2e/scenarios/prompt-decisions/scenario.ts b/e2e/scenarios/prompt-decisions/scenario.ts new file mode 100644 index 000000000..ae18ad3bb --- /dev/null +++ b/e2e/scenarios/prompt-decisions/scenario.ts @@ -0,0 +1,57 @@ +import { initLogger, projects } from "braintrust"; +import { + getTestRunId, + runMain, + scopedName, +} from "../../helpers/scenario-runtime"; + +async function main() { + const logger = initLogger({ + projectName: scopedName("e2e-prompt-decisions"), + }); + const project = projects.create({ name: scopedName("e2e-prompt-decisions") }); + const prompt = project.prompts.create({ + name: "decisions", + model: "decision-model", + input: [{ role: "user", content: "Evaluate {{text}}" }], + questions: [ + { type: "predicate", instructions: "Is {{text}} a greeting?" }, + { + type: "choice", + instructions: "Choose the category", + choices: [{ value: "{{category}}" }, { value: false }], + }, + { + type: "score", + instructions: "Rate clarity", + levels: [{ label: "{{level}}", description: "Very {{level}}" }], + }, + ], + }); + const variables = { text: "hello", category: "greeting", level: "clear" }; + await logger.traced( + async (span) => { + const built = await prompt.buildWithAttachments(variables, { + flavor: "decisions", + strict: true, + }); + span.log({ + output: { + model: built.model, + input: built.input, + questions: built.questions, + }, + }); + }, + { + name: "prompt-decisions", + event: { + input: variables, + metadata: { scenario: "prompt-decisions", testRunId: getTestRunId() }, + }, + }, + ); + await logger.flush(); +} + +runMain(main); diff --git a/js/src/cli/util/pull.ts b/js/src/cli/util/pull.ts index 76d214237..bff7b1253 100644 --- a/js/src/cli/util/pull.ts +++ b/js/src/cli/util/pull.ts @@ -1,9 +1,9 @@ import { - Function as functionSchema, - type FunctionType as FunctionObject, + Function as generatedFunctionSchema, type SavedFunctionIdType as SavedFunctionId, type ToolFunctionDefinitionType as ToolFunctionDefinition, } from "../../generated_types"; +import { promptDataSchema } from "../../prompt-schemas"; import { _internalGetGlobalState } from "../../logger"; import { loadCLIEnv } from "./bundle"; import { PullArgs } from "./types"; @@ -22,6 +22,11 @@ import { } from "../../framework2"; import pluralize from "pluralize"; +const functionSchema = generatedFunctionSchema.extend({ + prompt_data: promptDataSchema.nullish(), +}); +type FunctionObject = z.infer; + export async function pullCommand(args: PullArgs) { await loadCLIEnv(args); @@ -240,10 +245,14 @@ function makeFunctionDefinition({ } const objectType = "prompt"; const prompt = func.prompt_data.prompt; - const promptContents = - prompt.type === "completion" - ? `prompt: ${doubleQuote(prompt.content)}` - : `messages: ${safeStringify(prompt.messages).trimStart()}`; + let promptContents: string; + if (prompt.type === "completion") { + promptContents = `prompt: ${doubleQuote(prompt.content)}`; + } else if (prompt.type === "decisions") { + promptContents = `input: ${safeStringify(prompt.input).trimStart()},\n questions: ${safeStringify(prompt.questions).trimStart()}`; + } else { + promptContents = `messages: ${safeStringify(prompt.messages).trimStart()}`; + } const rawToolsParsed = prompt.type === "chat" && prompt.tools && prompt.tools.length > 0 diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index 433b9991b..a86d9cb44 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -6,8 +6,8 @@ import { Prompt, RemoteEvalParameters } from "./logger"; import { promptDefinitionWithToolsSchema, promptDefinitionToPromptData, + promptDataSchema, } from "./prompt-schemas"; -import { PromptData as promptDataSchema } from "./generated_types"; import type { ZodSchema } from "./zod/utils"; // Schema for evaluation parameters diff --git a/js/src/framework2.test.ts b/js/src/framework2.test.ts index 35297ab36..d7bb0dc3b 100644 --- a/js/src/framework2.test.ts +++ b/js/src/framework2.test.ts @@ -19,6 +19,34 @@ afterEach(() => { vi.restoreAllMocks(); }); +test("decision scorers preserve questions and classifier settings", () => { + const project = projects.create({ name: "test-decision-scorers" }); + const addPrompt = vi.spyOn(project, "addPrompt"); + const questions = [ + { type: "predicate" as const, instructions: "Is {{output}} correct?" }, + ]; + project.scorers.create({ + name: "correctness", + model: "decision-model", + input: "{{input}}", + questions, + params: { temperature: 0 }, + useCot: false, + choiceScores: { true: 1, false: 0 }, + templateFormat: "none", + }); + expect(addPrompt.mock.calls[0][0].prompt).toEqual({ + prompt: { type: "decisions", input: "{{input}}", questions }, + options: { model: "decision-model", params: { temperature: 0 } }, + parser: { + type: "llm_classifier", + use_cot: false, + choice_scores: { true: 1, false: 0 }, + }, + template_format: "none", + }); +}); + function mockProjectRegister() { const postJson = vi.fn(async () => ({ project: { diff --git a/js/src/framework2.ts b/js/src/framework2.ts index 458f1d278..b90528e8a 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -8,8 +8,6 @@ import { type FunctionTypeEnumType as FunctionType, type IfExistsType as IfExists, type SavedFunctionIdType as SavedFunctionId, - type PromptBlockDataType as PromptBlockData, - type PromptDataType as PromptData, ToolFunctionDefinition as toolFunctionDefinitionSchema, type ToolFunctionDefinitionType as ToolFunctionDefinition, FunctionData as functionDataSchema, @@ -28,6 +26,7 @@ import type { BaseFnOpts, GenericFunction } from "./framework-types"; import type { EvalParameters } from "./eval-parameters"; import { promptDefinitionToPromptData, + type PromptData, type PromptDefinition, } from "./prompt-schemas"; import { zodToJsonSchema, type ZodSchema } from "./zod/utils"; @@ -265,30 +264,13 @@ export class ScorerBuilder { }); this.project.addCodeFunction(scorer); } else { - const promptBlock: PromptBlockData = - "messages" in opts - ? { - type: "chat", - messages: opts.messages, - } - : { - type: "completion", - content: opts.prompt, - }; const promptData: PromptData = { - prompt: promptBlock, - options: { - model: opts.model, - params: opts.params, - }, + ...promptDefinitionToPromptData(opts), parser: { type: "llm_classifier", use_cot: opts.useCot, choice_scores: opts.choiceScores, }, - ...(opts.templateFormat - ? { template_format: opts.templateFormat } - : {}), }; const codePrompt = new CodePrompt( this.project, diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index f64426a84..d609e4b5a 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -2321,6 +2321,38 @@ describe("prompt and parameters loaders", () => { expect(getJson.mock.calls[0][1]).not.toHaveProperty("environment"); }); + test("loadPrompt accepts saved decisions prompts", async () => { + getJson.mockResolvedValue({ + objects: [ + { + ...promptRow, + prompt_data: { + options: { model: "decision-model" }, + prompt: { + type: "decisions", + input: "{{text}}", + questions: [ + { type: "predicate", instructions: "Is {{text}} a greeting?" }, + ], + }, + }, + }, + ], + }); + const prompt = await loadPrompt({ + projectName: "test-project", + slug: "decisions-prompt", + state, + }); + expect( + prompt.build({ text: "hello" }, { flavor: "decisions" }), + ).toMatchObject({ + model: "decision-model", + input: "hello", + questions: [{ type: "predicate", instructions: "Is hello a greeting?" }], + }); + }); + test("loadPrompt prefers version over environment for id lookup", async () => { getJson.mockResolvedValue(promptRow); diff --git a/js/src/logger.ts b/js/src/logger.ts index c71af1444..5efe8c383 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -94,17 +94,23 @@ import { type ChatCompletionOpenAIMessageParamType as OpenAIMessage, DatasetSnapshot as datasetSnapshotSchema, type DatasetSnapshotType as DatasetSnapshot, - PromptData as promptDataSchema, - type PromptDataType as PromptData, - Prompt as promptSchema, - type PromptType as PromptRow, + Prompt as generatedPromptSchema, type PromptSessionEventType as PromptSessionEvent, type RepoInfoType as RepoInfo, type ObjectReferenceType as ObjectReference, - type PromptBlockDataType as PromptBlockData, type ResponseFormatJsonSchemaType as ResponseFormatJsonSchema, type ObjectReferenceType, } from "./generated_types"; +import { + promptDataSchema, + type PromptBlockData, + type PromptData, +} from "./prompt-schemas"; + +const promptSchema = generatedPromptSchema.extend({ + prompt_data: promptDataSchema.nullish(), +}); +type PromptRow = z.infer; const BRAINTRUST_ATTACHMENT = BraintrustAttachmentReferenceSchema.shape.type.value; @@ -9795,7 +9801,7 @@ export type CompletionPrompt = { prompt: string; }; -export type CompiledPrompt = +export type CompiledPrompt = CompiledPromptParams & { span_info?: { name?: string; @@ -9813,8 +9819,10 @@ export type CompiledPrompt = ? ChatPrompt : Flavor extends "completion" ? CompletionPrompt - : // eslint-disable-next-line @typescript-eslint/no-empty-object-type - {}); + : Pick< + Extract, + "input" | "questions" + >); export type DefaultPromptArgs = Partial< CompiledPromptParams & AnyModelParam & ChatPrompt & CompletionPrompt @@ -10156,7 +10164,7 @@ export class Prompt< * * @param buildArgs Args to forward along to the prompt template. */ - public build( + public build( buildArgs: unknown, options: { flavor?: Flavor; @@ -10182,7 +10190,7 @@ export class Prompt< * @param buildArgs Args to forward along to the prompt template. */ public async buildWithAttachments< - Flavor extends "chat" | "completion" = "chat", + Flavor extends PromptBlockData["type"] = "chat", >( buildArgs: unknown, options: { @@ -10206,7 +10214,7 @@ export class Prompt< }) as CompiledPrompt; } - private runBuild( + private runBuild( buildArgs: unknown, options: { flavor: Flavor; @@ -10283,19 +10291,21 @@ export class Prompt< options: { ...options, templateFormat: resolvedTemplateFormat }, }); - if (flavor === "chat") { - if (renderedPrompt.type !== "chat") { - throw new Error( - "Prompt is a completion prompt. Use buildCompletion() instead", - ); - } + if (renderedPrompt.type !== flavor) { + throw new Error( + `Prompt is a ${renderedPrompt.type} prompt. Use flavor: '${renderedPrompt.type}' instead`, + ); + } + + const renderedParams = renderPromptParams(params, variables, { + strict: options.strict, + templateFormat: resolvedTemplateFormat, + }); + if (renderedPrompt.type === "chat") { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return { - ...renderPromptParams(params, variables, { - strict: options.strict, - templateFormat: resolvedTemplateFormat, - }), + ...renderedParams, ...spanInfo, messages: renderedPrompt.messages, ...(renderedPrompt.tools @@ -10306,20 +10316,21 @@ export class Prompt< } : undefined), } as CompiledPrompt; - } else if (flavor === "completion") { - if (renderedPrompt.type !== "completion") { - throw new Error(`Prompt is a chat prompt. Use flavor: 'chat' instead`); - } - + } else if (renderedPrompt.type === "completion") { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return { - ...renderPromptParams(params, variables, { - strict: options.strict, - templateFormat: resolvedTemplateFormat, - }), + ...renderedParams, ...spanInfo, prompt: renderedPrompt.content, } as CompiledPrompt; + } else if (renderedPrompt.type === "decisions") { + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return { + ...renderedParams, + ...spanInfo, + input: renderedPrompt.input, + questions: renderedPrompt.questions, + } as CompiledPrompt; } else { throw new Error("never!"); } @@ -10359,14 +10370,13 @@ export class Prompt< }; const templateFormat = parseTemplateFormat(options.templateFormat); + const render = (template: string) => + renderTemplateContent(template, variables, escape, { + strict: options.strict, + templateFormat, + }); if (prompt.type === "chat") { - const render = (template: string) => - renderTemplateContent(template, variables, escape, { - strict: options.strict, - templateFormat: templateFormat, - }); - const baseMessages = (prompt.messages || []).map((m) => renderMessageImpl(render, m, variables), ); @@ -10395,14 +10405,65 @@ export class Prompt< ); } - const content = renderTemplateContent(prompt.content, variables, escape, { - strict: options.strict, - templateFormat: templateFormat, - }); return { type: "completion", - content, + content: render(prompt.content), }; + } else if (prompt.type === "decisions") { + if (options.messages) { + throw new Error( + "extra messages are not supported for decisions prompts", + ); + } + + let input: typeof prompt.input; + if (typeof prompt.input === "string") { + input = render(prompt.input); + } else { + input = prompt.input.map((message) => ({ + ...message, + content: + typeof message.content === "string" + ? render(message.content) + : message.content.map((part) => ({ + ...part, + text: render(part.text), + })), + })); + } + + const questions = prompt.questions.map((question) => { + const renderedQuestion = { ...question }; + if (question.name !== undefined) { + renderedQuestion.name = render(question.name); + } + renderedQuestion.instructions = render(question.instructions); + + if (renderedQuestion.type === "choice") { + renderedQuestion.choices = renderedQuestion.choices.map((choice) => ({ + ...choice, + value: + typeof choice.value === "string" + ? render(choice.value) + : choice.value, + ...(choice.description !== undefined + ? { description: render(choice.description) } + : {}), + })); + } else if (renderedQuestion.type === "score") { + renderedQuestion.levels = renderedQuestion.levels.map((level) => ({ + ...level, + label: render(level.label), + ...(level.description !== undefined + ? { description: render(level.description) } + : {}), + })); + } + + return renderedQuestion; + }); + + return { type: "decisions", input, questions }; } else { const _: never = prompt; throw new Error(`Invalid prompt type: ${_}`); diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts index c4203f90e..d9c2f88e8 100644 --- a/js/src/prompt-schemas.ts +++ b/js/src/prompt-schemas.ts @@ -4,10 +4,61 @@ import { type ToolFunctionDefinitionType as ToolFunctionDefinition, ChatCompletionMessageParam as chatCompletionMessageParamSchema, ModelParams as modelParamsSchema, - type PromptBlockDataType as PromptBlockData, - type PromptDataType as PromptData, + ChatCompletionContentPartText as chatCompletionContentPartTextSchema, + PromptBlockData as generatedPromptBlockDataSchema, + PromptData as generatedPromptDataSchema, } from "./generated_types"; +const decisionPromptSchema = z.object({ + type: z.literal("decisions"), + input: z.union([ + z.string(), + z.array( + z.object({ + role: z.enum(["system", "developer", "user", "assistant"]), + content: z.union([ + z.string(), + z.array(chatCompletionContentPartTextSchema), + ]), + }), + ), + ]), + questions: z.array( + z.object({ name: z.string().optional(), instructions: z.string() }).and( + z.discriminatedUnion("type", [ + z.object({ type: z.literal("predicate") }), + z.object({ + type: z.literal("choice"), + choices: z.array( + z.object({ + value: z.union([z.string(), z.boolean()]), + description: z.string().optional(), + }), + ), + }), + z.object({ + type: z.literal("score"), + levels: z.array( + z.object({ label: z.string(), description: z.string().optional() }), + ), + }), + ]), + ), + ), +}); + +// Keep SDK prompt support independent of generated platform schema updates. +const promptBlockDataSchema = z.union([ + generatedPromptBlockDataSchema, + decisionPromptSchema, +]); +export type PromptBlockData = z.infer; + +export const promptDataSchema = generatedPromptDataSchema.extend({ + prompt: promptBlockDataSchema.nullish(), +}); +export type PromptData = z.infer; + // This roughly maps to promptBlockDataSchema, but is more ergonomic for the user. export const promptContentsSchema = z.union([ z.object({ @@ -16,6 +67,7 @@ export const promptContentsSchema = z.union([ z.object({ messages: z.array(chatCompletionMessageParamSchema), }), + decisionPromptSchema.omit({ type: true }), ]); export type PromptContents = z.infer; @@ -45,20 +97,26 @@ export function promptDefinitionToPromptData( promptDefinition: PromptDefinition, rawTools?: ToolFunctionDefinition[], ): PromptData { - const promptBlock: PromptBlockData = - "messages" in promptDefinition - ? { - type: "chat", - messages: promptDefinition.messages, - tools: - rawTools && rawTools.length > 0 - ? JSON.stringify(rawTools) - : undefined, - } - : { - type: "completion", - content: promptDefinition.prompt, - }; + let promptBlock: PromptBlockData; + if ("messages" in promptDefinition) { + promptBlock = { + type: "chat", + messages: promptDefinition.messages, + tools: + rawTools && rawTools.length > 0 ? JSON.stringify(rawTools) : undefined, + }; + } else if ("questions" in promptDefinition) { + promptBlock = { + type: "decisions", + input: promptDefinition.input, + questions: promptDefinition.questions, + }; + } else { + promptBlock = { + type: "completion", + content: promptDefinition.prompt, + }; + } return { prompt: promptBlock, diff --git a/js/src/prompt.test.ts b/js/src/prompt.test.ts index 3e18470ff..f3db3af24 100644 --- a/js/src/prompt.test.ts +++ b/js/src/prompt.test.ts @@ -1,7 +1,12 @@ import { beforeAll, describe, test, expect, vi } from "vitest"; import { configureNode } from "./node/config"; import { Prompt } from "./logger"; -import { type PromptDataType as PromptData } from "./generated_types"; +import { validateParameters } from "./eval-parameters"; +import { + type PromptData, + promptDefinitionToPromptData, + promptDefinitionSchema, +} from "./prompt-schemas"; describe("prompt strict mode", () => { test("strict mode", () => { @@ -346,3 +351,250 @@ describe("prompt template_format (unconfigured/browser-like)", () => { ); }); }); + +describe("decisions prompts", () => { + const definition = { + model: "decision-model", + input: "Evaluate {{text}}", + questions: [ + { + type: "predicate" as const, + name: "{{name}}", + instructions: "Is {{text}} safe?", + }, + { + type: "choice" as const, + instructions: "Choose {{topic}}", + choices: [ + { value: "{{topic}}", description: "About {{topic}}" }, + { value: false }, + ], + }, + { + type: "score" as const, + instructions: "Rate {{text}}", + levels: [ + { label: "{{level}}", description: "Very {{level}}" }, + { label: "low" }, + ], + }, + ], + }; + const args = { + text: "hello", + name: "safe", + topic: "greeting", + level: "high", + }; + + test("accepts decisions prompts as eval parameter values", async () => { + const schema = { prompt: { type: "prompt" as const } }; + const { prompt } = await validateParameters( + { prompt: promptDefinitionToPromptData(definition) }, + schema, + ); + expect(prompt.build(args, { flavor: "decisions" }).input).toBe( + "Evaluate hello", + ); + }); + + test("builds code-defined prompts and preserves question value types", () => { + const data = promptDefinitionToPromptData( + promptDefinitionSchema.parse(definition), + ); + const original = structuredClone(data); + const prompt = Prompt.fromPromptData("decisions", data); + const built = prompt.build(args, { flavor: "decisions", strict: true }); + expect(built).toMatchObject({ + model: "decision-model", + input: "Evaluate hello", + questions: [ + { type: "predicate", name: "safe", instructions: "Is hello safe?" }, + { + type: "choice", + instructions: "Choose greeting", + choices: [ + { value: "greeting", description: "About greeting" }, + { value: false }, + ], + }, + { + type: "score", + instructions: "Rate hello", + levels: [ + { label: "high", description: "Very high" }, + { label: "low" }, + ], + }, + ], + }); + expect(built).not.toHaveProperty("messages"); + expect(built).not.toHaveProperty("type"); + expect(data).toEqual(original); + }); + + test("renders message content while preserving roles and cache control", () => { + const prompt = Prompt.fromPromptData("decisions", { + options: { model: "decision-model" }, + prompt: { + type: "decisions", + input: [ + { role: "system", content: "Evaluate {{topic}}" }, + { role: "developer", content: "Be {{level}} quality" }, + { + role: "user", + content: [ + { + type: "text", + text: "{{text}}", + cache_control: { type: "ephemeral" }, + }, + ], + }, + { role: "assistant", content: "{{name}}" }, + ], + questions: [], + }, + }); + expect(prompt.build(args, { flavor: "decisions" }).input).toEqual([ + { role: "system", content: "Evaluate greeting" }, + { role: "developer", content: "Be high quality" }, + { + role: "user", + content: [ + { type: "text", text: "hello", cache_control: { type: "ephemeral" } }, + ], + }, + { role: "assistant", content: "safe" }, + ]); + }); + + test.each([ + { input: "{{missing}}", questions: [] }, + { + input: [ + { + role: "user" as const, + content: [{ type: "text" as const, text: "{{missing}}" }], + }, + ], + questions: [], + }, + { + input: "text", + questions: [{ type: "predicate" as const, instructions: "{{missing}}" }], + }, + { + input: "text", + questions: [ + { + type: "choice" as const, + instructions: "choose", + choices: [{ value: false, description: "{{missing}}" }], + }, + ], + }, + { + input: "text", + questions: [ + { + type: "score" as const, + instructions: "rate", + levels: [{ label: "{{missing}}" }], + }, + ], + }, + ])("applies strict mode to input and questions: %j", (contents) => { + const prompt = Prompt.fromPromptData("decisions", { + options: { model: "decision-model" }, + prompt: { type: "decisions", ...contents }, + }); + expect(() => + prompt.build({}, { flavor: "decisions", strict: true }), + ).toThrow(); + expect(() => + prompt.build({}, { flavor: "decisions", strict: false }), + ).not.toThrow(); + }); + + test("honors saved template format and build overrides", () => { + const prompt = Prompt.fromPromptData("decisions", { + ...promptDefinitionToPromptData(definition), + template_format: "none", + }); + expect(prompt.build(args, { flavor: "decisions" })).toMatchObject( + definition, + ); + expect( + prompt.build(args, { flavor: "decisions", templateFormat: "mustache" }) + .input, + ).toBe("Evaluate hello"); + }); + + test("uses the shared renderer for nunjucks", () => { + const prompt = Prompt.fromPromptData( + "decisions", + promptDefinitionToPromptData(definition), + ); + expect(() => + prompt.build(args, { flavor: "decisions", templateFormat: "nunjucks" }), + ).toThrow("Nunjucks templating requires"); + }); + + test("requires an explicit matching flavor and rejects extra messages", () => { + const prompt = Prompt.fromPromptData( + "decisions", + promptDefinitionToPromptData(definition), + ); + expect(() => prompt.build(args)).toThrow("Use flavor: 'decisions' instead"); + expect(() => prompt.build(args, { flavor: "completion" })).toThrow( + "Use flavor: 'decisions' instead", + ); + expect(() => + prompt.build(args, { + flavor: "decisions", + messages: [{ role: "user", content: "extra" }], + }), + ).toThrow("extra messages are not supported for decisions prompts"); + const chat = Prompt.fromPromptData("chat", { + options: { model: "chat-model" }, + prompt: { type: "chat", messages: [] }, + }); + expect(() => chat.build({}, { flavor: "decisions" })).toThrow( + "Use flavor: 'chat' instead", + ); + }); + + test.each([true, false])( + "preserves prompt metadata and noTrace=%s in both build methods", + async (noTrace) => { + const prompt = new Prompt( + { + id: "prompt-id", + project_id: "project-id", + _xact_id: "version", + name: "decisions", + slug: "decisions", + prompt_data: promptDefinitionToPromptData(definition), + }, + { temperature: 0.2 }, + noTrace, + ); + const built = prompt.build(args, { flavor: "decisions" }); + expect(built.temperature).toBe(0.2); + expect( + await prompt.buildWithAttachments(args, { flavor: "decisions" }), + ).toEqual(built); + if (noTrace) { + expect(built).not.toHaveProperty("span_info"); + } else { + expect(built.span_info?.metadata.prompt).toEqual({ + id: "prompt-id", + project_id: "project-id", + version: "version", + variables: args, + }); + } + }, + ); +}); From 6d1de6fca7ba3fcd7888aed3c02d210852b7dd59 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:02:24 +0000 Subject: [PATCH 2/2] Update PR #2769 --- .changeset/add-decisions-prompt-support.md | 5 ++ js/dev/types.ts | 2 +- js/src/logger.ts | 9 +-- js/src/prompt-schemas.ts | 15 +++- js/src/prompt.test.ts | 81 ++++++++++++++++++---- 5 files changed, 91 insertions(+), 21 deletions(-) create mode 100644 .changeset/add-decisions-prompt-support.md diff --git a/.changeset/add-decisions-prompt-support.md b/.changeset/add-decisions-prompt-support.md new file mode 100644 index 000000000..20008c653 --- /dev/null +++ b/.changeset/add-decisions-prompt-support.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Add decisions prompt support diff --git a/js/dev/types.ts b/js/dev/types.ts index bac98a5ac..9d0788218 100644 --- a/js/dev/types.ts +++ b/js/dev/types.ts @@ -2,8 +2,8 @@ import { FunctionId as functionIdSchema, InvokeParent as invokeParentSchema, RunEval as runEvalSchema, - PromptData as promptDataSchema, } from "../src/generated_types"; +import { promptDataSchema } from "../src/prompt-schemas"; import { z } from "zod/v3"; import { EvaluatorDef } from "../src/framework"; import { BaseMetadata } from "../src/logger"; diff --git a/js/src/logger.ts b/js/src/logger.ts index 5efe8c383..85ffea9de 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -10425,10 +10425,11 @@ export class Prompt< content: typeof message.content === "string" ? render(message.content) - : message.content.map((part) => ({ - ...part, - text: render(part.text), - })), + : message.content.map((part) => + part.type === "input_text" + ? { ...part, text: render(part.text) } + : { ...part, image_url: render(part.image_url) }, + ), })); } diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts index d9c2f88e8..ea1f2ddc6 100644 --- a/js/src/prompt-schemas.ts +++ b/js/src/prompt-schemas.ts @@ -4,7 +4,6 @@ import { type ToolFunctionDefinitionType as ToolFunctionDefinition, ChatCompletionMessageParam as chatCompletionMessageParamSchema, ModelParams as modelParamsSchema, - ChatCompletionContentPartText as chatCompletionContentPartTextSchema, PromptBlockData as generatedPromptBlockDataSchema, PromptData as generatedPromptDataSchema, } from "./generated_types"; @@ -15,10 +14,20 @@ const decisionPromptSchema = z.object({ z.string(), z.array( z.object({ - role: z.enum(["system", "developer", "user", "assistant"]), + role: z.literal("user"), + type: z.literal("message").optional(), content: z.union([ z.string(), - z.array(chatCompletionContentPartTextSchema), + z.array( + z.discriminatedUnion("type", [ + z.object({ type: z.literal("input_text"), text: z.string() }), + z.object({ + type: z.literal("input_image"), + image_url: z.string(), + detail: z.string().nullish(), + }), + ]), + ), ]), }), ), diff --git a/js/src/prompt.test.ts b/js/src/prompt.test.ts index f3db3af24..88d20e2d9 100644 --- a/js/src/prompt.test.ts +++ b/js/src/prompt.test.ts @@ -1,7 +1,10 @@ -import { beforeAll, describe, test, expect, vi } from "vitest"; +import { beforeAll, describe, test, expect, expectTypeOf, vi } from "vitest"; import { configureNode } from "./node/config"; import { Prompt } from "./logger"; import { validateParameters } from "./eval-parameters"; +import { evaluatorDefinitionsSchema } from "../dev/types"; +import { serializeRemoteEvalParametersContainer } from "./framework2"; +import type { OpenAIDecisionCreateParams } from "./vendor-sdk-types/openai-decisions"; import { type PromptData, promptDefinitionToPromptData, @@ -398,6 +401,27 @@ describe("decisions prompts", () => { ); }); + test("round-trips decisions prompt defaults through evaluator manifests", () => { + const parameters = serializeRemoteEvalParametersContainer({ + prompt: { type: "prompt", default: definition }, + }); + const manifest = { evaluator: { parameters } }; + const parsed = evaluatorDefinitionsSchema.parse(manifest); + expect(parsed).toEqual(manifest); + const serialized = parsed.evaluator.parameters; + if (serialized?.type !== "braintrust.staticParameters") { + throw new Error("Expected static parameters"); + } + const parameter = serialized.schema.prompt; + if (parameter.type !== "prompt" || !parameter.default) { + throw new Error("Expected a prompt default"); + } + const prompt = Prompt.fromPromptData("decisions", parameter.default); + expect(prompt.build(args, { flavor: "decisions" }).input).toBe( + "Evaluate hello", + ); + }); + test("builds code-defined prompts and preserves question value types", () => { const data = promptDefinitionToPromptData( promptDefinitionSchema.parse(definition), @@ -405,6 +429,7 @@ describe("decisions prompts", () => { const original = structuredClone(data); const prompt = Prompt.fromPromptData("decisions", data); const built = prompt.build(args, { flavor: "decisions", strict: true }); + expectTypeOf(built).toMatchTypeOf(); expect(built).toMatchObject({ model: "decision-model", input: "Evaluate hello", @@ -433,49 +458,79 @@ describe("decisions prompts", () => { expect(data).toEqual(original); }); - test("renders message content while preserving roles and cache control", () => { + test("renders decisions messages with provider-compatible text and image parts", () => { const prompt = Prompt.fromPromptData("decisions", { options: { model: "decision-model" }, prompt: { type: "decisions", input: [ - { role: "system", content: "Evaluate {{topic}}" }, - { role: "developer", content: "Be {{level}} quality" }, + { role: "user", content: "Evaluate {{topic}}" }, { role: "user", + type: "message", content: [ { - type: "text", + type: "input_text", text: "{{text}}", - cache_control: { type: "ephemeral" }, + }, + { + type: "input_image", + image_url: "https://example.com/{{topic}}.png", + detail: "low", }, ], }, - { role: "assistant", content: "{{name}}" }, ], questions: [], }, }); - expect(prompt.build(args, { flavor: "decisions" }).input).toEqual([ - { role: "system", content: "Evaluate greeting" }, - { role: "developer", content: "Be high quality" }, + const built = prompt.build(args, { flavor: "decisions" }); + expectTypeOf(built).toMatchTypeOf(); + expect(built.input).toEqual([ + { role: "user", content: "Evaluate greeting" }, { role: "user", + type: "message", content: [ - { type: "text", text: "hello", cache_control: { type: "ephemeral" } }, + { type: "input_text", text: "hello" }, + { + type: "input_image", + image_url: "https://example.com/greeting.png", + detail: "low", + }, ], }, - { role: "assistant", content: "safe" }, ]); }); + test.each([ + { role: "system", content: "text" }, + { role: "developer", content: "text" }, + { role: "assistant", content: "text" }, + { role: "user", content: [{ type: "text", text: "text" }] }, + ])("rejects chat-only message formats: %j", (message) => { + expect( + promptDefinitionSchema.safeParse({ ...definition, input: [message] }) + .success, + ).toBe(false); + }); + test.each([ { input: "{{missing}}", questions: [] }, { input: [ { role: "user" as const, - content: [{ type: "text" as const, text: "{{missing}}" }], + content: [{ type: "input_text" as const, text: "{{missing}}" }], + }, + ], + questions: [], + }, + { + input: [ + { + role: "user" as const, + content: [{ type: "input_image" as const, image_url: "{{missing}}" }], }, ], questions: [],