feat: Add decisions prompt support

.changeset/add-decisions-prompt-support.md

@@ -0,0 +1,5 @@
+---
+"braintrust": minor
+---
+
+feat: Add decisions prompt support

e2e/config/pr-comment-scenarios.json

@@ -19,6 +19,11 @@
     "label": "Trace primitives and project caching",
     "metadataScenario": "trace-primitives-basic"
   },
+  {
+    "scenarioDirName": "prompt-decisions",
+    "label": "Decisions prompt building",
+    "metadataScenario": "prompt-decisions"
+  },
   {
     "scenarioDirName": "ts-exec-loader",
     "label": "TypeScript loader compatibility",

e2e/scenarios/prompt-decisions/scenario.test.ts

@@ -0,0 +1,35 @@
+import { expect, test } from "vitest";
+import {
+  prepareScenarioDir,
+  resolveScenarioDir,
+  withScenarioHarness,
+} from "../../helpers/scenario-harness";
+import { findLatestSpan } from "../../helpers/trace-selectors";
+
+const scenarioDir = await prepareScenarioDir({
+  scenarioDir: resolveScenarioDir(import.meta.url),
+});
+
+test("builds decisions prompts from the packaged SDK", async () => {
+  await withScenarioHarness(async ({ runScenarioDir, testRunEvents }) => {
+    await runScenarioDir({ scenarioDir });
+    const root = findLatestSpan(testRunEvents(), "prompt-decisions");
+    expect(root?.output).toEqual({
+      model: "decision-model",
+      input: [{ role: "user", content: "Evaluate hello" }],
+      questions: [
+        { type: "predicate", instructions: "Is hello a greeting?" },
+        {
+          type: "choice",
+          instructions: "Choose the category",
+          choices: [{ value: "greeting" }, { value: false }],
+        },
+        {
+          type: "score",
+          instructions: "Rate clarity",
+          levels: [{ label: "clear", description: "Very clear" }],
+        },
+      ],
+    });
+  });
+});

e2e/scenarios/prompt-decisions/scenario.ts

@@ -0,0 +1,57 @@
+import { initLogger, projects } from "braintrust";
+import {
+  getTestRunId,
+  runMain,
+  scopedName,
+} from "../../helpers/scenario-runtime";
+
+async function main() {
+  const logger = initLogger({
+    projectName: scopedName("e2e-prompt-decisions"),
+  });
+  const project = projects.create({ name: scopedName("e2e-prompt-decisions") });
+  const prompt = project.prompts.create({
+    name: "decisions",
+    model: "decision-model",
+    input: [{ role: "user", content: "Evaluate {{text}}" }],
+    questions: [
+      { type: "predicate", instructions: "Is {{text}} a greeting?" },
+      {
+        type: "choice",
+        instructions: "Choose the category",
+        choices: [{ value: "{{category}}" }, { value: false }],
+      },
+      {
+        type: "score",
+        instructions: "Rate clarity",
+        levels: [{ label: "{{level}}", description: "Very {{level}}" }],
+      },
+    ],
+  });
+  const variables = { text: "hello", category: "greeting", level: "clear" };
+  await logger.traced(
+    async (span) => {
+      const built = await prompt.buildWithAttachments(variables, {
+        flavor: "decisions",
+        strict: true,
+      });
+      span.log({
+        output: {
+          model: built.model,
+          input: built.input,
+          questions: built.questions,
+        },
+      });
+    },
+    {
+      name: "prompt-decisions",
+      event: {
+        input: variables,
+        metadata: { scenario: "prompt-decisions", testRunId: getTestRunId() },
+      },
+    },
+  );
+  await logger.flush();
+}
+
+runMain(main);

js/dev/types.ts

@@ -2,8 +2,8 @@ import {
   FunctionId as functionIdSchema,
   InvokeParent as invokeParentSchema,
   RunEval as runEvalSchema,
-  PromptData as promptDataSchema,
 } from "../src/generated_types";
+import { promptDataSchema } from "../src/prompt-schemas";
 import { z } from "zod/v3";
 import { EvaluatorDef } from "../src/framework";
 import { BaseMetadata } from "../src/logger";

js/src/cli/util/pull.ts

@@ -1,9 +1,9 @@
 import {
-  Function as functionSchema,
-  type FunctionType as FunctionObject,
+  Function as generatedFunctionSchema,
   type SavedFunctionIdType as SavedFunctionId,
   type ToolFunctionDefinitionType as ToolFunctionDefinition,
 } from "../../generated_types";
+import { promptDataSchema } from "../../prompt-schemas";
 import { _internalGetGlobalState } from "../../logger";
 import { loadCLIEnv } from "./bundle";
 import { PullArgs } from "./types";
@@ -22,6 +22,11 @@ import {
 } from "../../framework2";
 import pluralize from "pluralize";
 
+const functionSchema = generatedFunctionSchema.extend({
+  prompt_data: promptDataSchema.nullish(),
+});
+type FunctionObject = z.infer<typeof functionSchema>;
+
 export async function pullCommand(args: PullArgs) {
   await loadCLIEnv(args);
 
@@ -240,10 +245,14 @@ function makeFunctionDefinition({
   }
   const objectType = "prompt";
   const prompt = func.prompt_data.prompt;
-  const promptContents =
-    prompt.type === "completion"
-      ? `prompt: ${doubleQuote(prompt.content)}`
-      : `messages: ${safeStringify(prompt.messages).trimStart()}`;
+  let promptContents: string;
+  if (prompt.type === "completion") {
+    promptContents = `prompt: ${doubleQuote(prompt.content)}`;
+  } else if (prompt.type === "decisions") {
+    promptContents = `input: ${safeStringify(prompt.input).trimStart()},\n  questions: ${safeStringify(prompt.questions).trimStart()}`;
+  } else {
+    promptContents = `messages: ${safeStringify(prompt.messages).trimStart()}`;
+  }
 
   const rawToolsParsed =
     prompt.type === "chat" && prompt.tools && prompt.tools.length > 0

js/src/eval-parameters.ts

@@ -6,8 +6,8 @@
 import {
   promptDefinitionWithToolsSchema,
   promptDefinitionToPromptData,
+  promptDataSchema,
 } from "./prompt-schemas";
-import { PromptData as promptDataSchema } from "./generated_types";
 import type { ZodSchema } from "./zod/utils";
 
 // Schema for evaluation parameters
@@ -152,7 +152,7 @@
     throw Error(`Invalid parameters: ${errorMessages}`);
   }
 
   return rehydrateRemoteParameters(parameters, schema) as T;
 }
 
 function rehydrateRemoteParameters(

js/src/framework2.test.ts

@@ -19,6 +19,34 @@ afterEach(() => {
   vi.restoreAllMocks();
 });
 
+test("decision scorers preserve questions and classifier settings", () => {
+  const project = projects.create({ name: "test-decision-scorers" });
+  const addPrompt = vi.spyOn(project, "addPrompt");
+  const questions = [
+    { type: "predicate" as const, instructions: "Is {{output}} correct?" },
+  ];
+  project.scorers.create({
+    name: "correctness",
+    model: "decision-model",
+    input: "{{input}}",
+    questions,
+    params: { temperature: 0 },
+    useCot: false,
+    choiceScores: { true: 1, false: 0 },
+    templateFormat: "none",
+  });
+  expect(addPrompt.mock.calls[0][0].prompt).toEqual({
+    prompt: { type: "decisions", input: "{{input}}", questions },
+    options: { model: "decision-model", params: { temperature: 0 } },
+    parser: {
+      type: "llm_classifier",
+      use_cot: false,
+      choice_scores: { true: 1, false: 0 },
+    },
+    template_format: "none",
+  });
+});
+
 function mockProjectRegister() {
   const postJson = vi.fn(async () => ({
     project: {

js/src/framework2.ts

@@ -8,8 +8,6 @@ import {
   type FunctionTypeEnumType as FunctionType,
   type IfExistsType as IfExists,
   type SavedFunctionIdType as SavedFunctionId,
-  type PromptBlockDataType as PromptBlockData,
-  type PromptDataType as PromptData,
   ToolFunctionDefinition as toolFunctionDefinitionSchema,
   type ToolFunctionDefinitionType as ToolFunctionDefinition,
   FunctionData as functionDataSchema,
@@ -28,6 +26,7 @@ import type { BaseFnOpts, GenericFunction } from "./framework-types";
 import type { EvalParameters } from "./eval-parameters";
 import {
   promptDefinitionToPromptData,
+  type PromptData,
   type PromptDefinition,
 } from "./prompt-schemas";
 import { zodToJsonSchema, type ZodSchema } from "./zod/utils";
@@ -265,30 +264,13 @@ export class ScorerBuilder {
       });
       this.project.addCodeFunction(scorer);
     } else {
-      const promptBlock: PromptBlockData =
-        "messages" in opts
-          ? {
-              type: "chat",
-              messages: opts.messages,
-            }
-          : {
-              type: "completion",
-              content: opts.prompt,
-            };
       const promptData: PromptData = {
-        prompt: promptBlock,
-        options: {
-          model: opts.model,
-          params: opts.params,
-        },
+        ...promptDefinitionToPromptData(opts),
         parser: {
           type: "llm_classifier",
           use_cot: opts.useCot,
           choice_scores: opts.choiceScores,
         },
-        ...(opts.templateFormat
-          ? { template_format: opts.templateFormat }
-          : {}),
       };
       const codePrompt = new CodePrompt(
         this.project,

js/src/logger.test.ts

@@ -2321,6 +2321,38 @@ describe("prompt and parameters loaders", () => {
     expect(getJson.mock.calls[0][1]).not.toHaveProperty("environment");
   });
 
+  test("loadPrompt accepts saved decisions prompts", async () => {
+    getJson.mockResolvedValue({
+      objects: [
+        {
+          ...promptRow,
+          prompt_data: {
+            options: { model: "decision-model" },
+            prompt: {
+              type: "decisions",
+              input: "{{text}}",
+              questions: [
+                { type: "predicate", instructions: "Is {{text}} a greeting?" },
+              ],
+            },
+          },
+        },
+      ],
+    });
+    const prompt = await loadPrompt({
+      projectName: "test-project",
+      slug: "decisions-prompt",
+      state,
+    });
+    expect(
+      prompt.build({ text: "hello" }, { flavor: "decisions" }),
+    ).toMatchObject({
+      model: "decision-model",
+      input: "hello",
+      questions: [{ type: "predicate", instructions: "Is hello a greeting?" }],
+    });
+  });
+
   test("loadPrompt prefers version over environment for id lookup", async () => {
     getJson.mockResolvedValue(promptRow);