From 5ee72469d3446ea82a59c96153b3e230410d06c6 Mon Sep 17 00:00:00 2001 From: "braintrust-bot[bot]" <215900051+braintrust-bot[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:51:04 +0000 Subject: [PATCH] chore: Update platform types --- generated_types.json | 693 +++++++++++++++++++++++++++++-- js/src/generated_plain_types.ts | 690 +++++++++++++++++++++++++----- js/src/generated_types.ts | 203 ++++++++- js/util/generated_plain_types.ts | 690 +++++++++++++++++++++++++----- js/util/generated_types.ts | 203 ++++++++- 5 files changed, 2198 insertions(+), 281 deletions(-) diff --git a/generated_types.json b/generated_types.json index b8c224046..ed1440cd4 100644 --- a/generated_types.json +++ b/generated_types.json @@ -360,7 +360,8 @@ "description": "Name of the api key" }, "preview_name": { - "type": "string" + "type": "string", + "description": "Masked preview showing the key prefix and last four characters, e.g. sk-…AbCd." }, "user_id": { "type": ["string", "null"], @@ -403,6 +404,9 @@ }, "token": { "type": "string" + }, + "triggered_xact_id": { + "type": "string" } }, "required": ["kind"] @@ -1669,7 +1673,7 @@ }, "snapshot_ref": { "type": "string", - "description": "sandbox snapshot ref" + "description": "The Modal snapshot to start the sandbox from" } }, "required": ["provider", "snapshot_ref"] @@ -1830,7 +1834,7 @@ }, "span_id": { "type": "string", - "description": "A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing" + "description": "A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing" }, "root_span_id": { "type": "string", @@ -1973,7 +1977,7 @@ }, "xact_id": { "type": "string", - "description": "Transaction id of the brainstore version at the time of the snapshot" + "description": "Transaction ID of the dataset version at the time of the snapshot" }, "created": { "type": ["string", "null"], @@ -2054,7 +2058,7 @@ "automation_integration" ], "default": "env_var", - "description": "The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys" + "description": "The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys, sandbox_provider for sandbox provider credentials, automation_integration for automation delivery credentials" } }, "required": ["id", "object_type", "object_id", "name"] @@ -2205,7 +2209,7 @@ "minimum": 0, "maximum": 1 }, - "description": "A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments" + "description": "A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments." }, "metadata": { "type": ["object", "null"], @@ -2249,13 +2253,13 @@ "description": "The total number of tokens in the input and output of the experiment event." }, "caller_functionname": { - "description": "This metric is deprecated" + "description": "Deprecated. Use `context.caller_functionname` instead." }, "caller_filename": { - "description": "This metric is deprecated" + "description": "Deprecated. Use `context.caller_filename` instead." }, "caller_lineno": { - "description": "This metric is deprecated" + "description": "Deprecated. Use `context.caller_lineno` instead." } }, "additionalProperties": { @@ -2284,7 +2288,7 @@ }, "span_id": { "type": "string", - "description": "A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing" + "description": "A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing" }, "span_parents": { "type": ["array", "null"], @@ -2670,7 +2674,7 @@ }, "internal": { "type": ["boolean", "null"], - "description": "The function exists for internal purposes and should not be displayed in the list of functions." + "description": "If true, the function was created for use by another Braintrust object and is hidden from function lists in the Braintrust UI." } }, "required": ["object_type", "object_id"] @@ -2881,12 +2885,193 @@ "title": "topic_map" } ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["simulation"] + }, + "config": { + "anyOf": [ + { + "type": "object", + "properties": { + "initiator": { + "type": "string", + "enum": ["agent", "simulated_user"], + "default": "agent", + "description": "Who speaks first in the conversation" + }, + "timeout_seconds": { + "type": "integer", + "minimum": 15, + "maximum": 3600, + "default": 600, + "description": "Maximum conversation duration in seconds" + }, + "modality": { + "type": "string", + "enum": ["voice"], + "default": "voice" + }, + "agent": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["livekit"] + }, + "server_url": { + "type": "string", + "minLength": 1, + "description": "The WebSocket URL of the LiveKit project hosting the agent" + }, + "agent_name": { + "type": "string", + "minLength": 1, + "description": "The name of the LiveKit agent to dispatch" + } + }, + "required": ["type", "server_url", "agent_name"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["phone"] + }, + "phone_number": { + "type": "string", + "pattern": "^\\+[1-9]\\d{7,14}$" + } + }, + "required": ["type", "phone_number"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["sip"] + }, + "uri": { + "type": "string", + "pattern": "^sips?:[^\\s]+$" + } + }, + "required": ["type", "uri"], + "additionalProperties": false + } + ] + } + }, + "required": ["agent"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "initiator": { + "type": "string", + "enum": ["agent", "simulated_user"], + "default": "agent", + "description": "Who speaks first in the conversation" + }, + "timeout_seconds": { + "type": "integer", + "minimum": 15, + "maximum": 3600, + "default": 600, + "description": "Maximum conversation duration in seconds" + }, + "modality": { + "type": "string", + "enum": ["text"] + }, + "agent": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["livekit"] + }, + "server_url": { + "type": "string", + "minLength": 1, + "description": "The WebSocket URL of the LiveKit project hosting the agent" + }, + "agent_name": { + "type": "string", + "minLength": 1, + "description": "The name of the LiveKit agent to dispatch" + } + }, + "required": ["type", "server_url", "agent_name"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["acp"] + }, + "command": { + "type": "string", + "minLength": 1, + "description": "Executable for the ACP agent subprocess, communicating over stdio" + }, + "args": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "Arguments passed to the executable without shell interpretation" + }, + "cwd": { + "type": "string", + "pattern": "^(?:[/]|[A-Za-z]:[\\\\/]|\\\\\\\\)", + "description": "Absolute working directory for the ACP session on the execution host" + }, + "permissions": { + "type": "string", + "enum": ["deny"], + "default": "deny", + "description": "Deny ACP permission requests; this does not sandbox the subprocess" + } + }, + "required": ["type", "command", "cwd"], + "additionalProperties": false + } + ] + } + }, + "required": ["modality", "agent"], + "additionalProperties": false + } + ] + } + }, + "required": ["type", "config"], + "additionalProperties": false, + "description": "A task function that runs a conversation between an agent and a simulated user.", + "title": "simulation" } ] }, "FunctionFormat": { "type": "string", - "enum": ["llm", "code", "global", "graph", "topic_map"] + "enum": ["llm", "code", "global", "graph", "topic_map", "simulation"] }, "FunctionId": { "anyOf": [ @@ -3557,6 +3742,10 @@ { "type": "object", "properties": { + "request_source": { + "type": "string", + "description": "A label identifying the caller, used to attribute usage" + }, "input": { "description": "Argument to the function, which can be any JSON serializable value" }, @@ -3566,7 +3755,7 @@ "metadata": { "type": ["object", "null"], "additionalProperties": {}, - "description": "Any relevant metadata. This will be logged and available as the `metadata` argument." + "description": "Any relevant metadata to log on the span. For prompts, this is available as the `metadata` template variable. For code functions, this is passed through hooks and is not added to `input`." }, "tags": { "type": ["array", "null"], @@ -3653,7 +3842,7 @@ } }, "required": ["id", "span_id", "root_span_id"], - "description": "Identifiers for the row to to log a subspan under" + "description": "Identifiers for the row to log a subspan under" }, "propagated_event": { "type": ["object", "null"], @@ -4112,6 +4301,11 @@ "status": { "$ref": "#/components/schemas/AutomationStatus" }, + "service_tier": { + "type": "string", + "enum": ["standard", "flex"], + "description": "Service tier for online scoring. If omitted, prompt-level service tiers are preserved." + }, "sampling_rate": { "type": "number", "minimum": 0, @@ -4177,19 +4371,24 @@ "description": "Name of the organization" }, "api_url": { - "type": ["string", "null"] + "type": ["string", "null"], + "description": "URL of the organization's data plane API. If unset, the Braintrust-hosted API is used." }, "is_universal_api": { - "type": ["boolean", "null"] + "type": ["boolean", "null"], + "description": "Whether the data plane at `api_url` also serves the AI proxy, at `api_url` with `/v1/proxy` appended" }, "is_dataplane_private": { - "type": ["boolean", "null"] + "type": ["boolean", "null"], + "description": "Whether the organization's data plane is only reachable from a private network" }, "proxy_url": { - "type": ["string", "null"] + "type": ["string", "null"], + "description": "URL of the organization's AI proxy" }, "realtime_url": { - "type": ["string", "null"] + "type": ["string", "null"], + "description": "URL of the organization's realtime service" }, "created": { "type": ["string", "null"], @@ -4198,11 +4397,13 @@ }, "archived_at": { "type": ["string", "null"], - "format": "date-time" + "format": "date-time", + "description": "Date the organization was archived" }, "deleted_at": { "type": ["string", "null"], - "format": "date-time" + "format": "date-time", + "description": "Date the organization was deleted" }, "image_rendering_mode": { "$ref": "#/components/schemas/ImageRenderingMode" @@ -4276,7 +4477,7 @@ "update_acls", "delete_acls" ], - "description": "Each permission permits a certain type of operation on an object in the system\n\nPermissions can be assigned to to objects on an individual basis, or grouped into roles" + "description": "Each permission permits a certain type of operation on an object in the system\n\nPermissions can be assigned to objects on an individual basis, or grouped into roles" }, "PreprocessorId": { "anyOf": [ @@ -4483,7 +4684,7 @@ "type": "string", "minLength": 1, "maxLength": 10000, - "description": "Publish a Slack mrkdwn digest.\n\nPattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say \"No current pattern outcomes.\" This requirement overrides conflicting formatting guidance.\n\nInclude a complete \"*Pattern outcomes*\" section with one row for every included Pattern.\n\nUse this row format exactly:\n• — `outcome`\n\nIf a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern.\n\nAfter the outcome list, include a \"*Highlights*\" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report." + "description": "Instructions for Loop to format content sent to this destination" } }, "required": ["type", "workspace_id", "channel"] @@ -4805,7 +5006,7 @@ "type": "string", "minLength": 1, "maxLength": 10000, - "description": "Publish a Slack mrkdwn digest.\n\nPattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say \"No current pattern outcomes.\" This requirement overrides conflicting formatting guidance.\n\nInclude a complete \"*Pattern outcomes*\" section with one row for every included Pattern.\n\nUse this row format exactly:\n• — `outcome`\n\nIf a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern.\n\nAfter the outcome list, include a \"*Highlights*\" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report." + "description": "Instructions for Loop to format content sent to this destination" } }, "required": ["type", "workspace_id", "channel"] @@ -4915,11 +5116,11 @@ "type": "string", "format": "uuid" }, - "description": "Sorted ids of active projects in this project group" + "description": "Sorted ids of active projects in this project group. Modifying membership requires project_group:update on the group." } }, "required": ["id", "org_id", "name", "member_projects"], - "description": "A project group is a collection of projects which can inherit access from project-group ACLs" + "description": "A project group is a collection of projects which inherit access from project_group_projects ACLs. The group's definition and membership are governed separately by project_group ACLs." }, "ProjectLogsEvent": { "type": "object", @@ -4975,7 +5176,7 @@ "minimum": 0, "maximum": 1 }, - "description": "A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs." + "description": "A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs." }, "metadata": { "type": ["object", "null"], @@ -5019,13 +5220,13 @@ "description": "The total number of tokens in the input and output of the project logs event." }, "caller_functionname": { - "description": "This metric is deprecated" + "description": "Deprecated. Use `context.caller_functionname` instead." }, "caller_filename": { - "description": "This metric is deprecated" + "description": "Deprecated. Use `context.caller_filename` instead." }, "caller_lineno": { - "description": "This metric is deprecated" + "description": "Deprecated. Use `context.caller_lineno` instead." } }, "additionalProperties": { @@ -5054,7 +5255,7 @@ }, "span_id": { "type": "string", - "description": "A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing" + "description": "A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing" }, "span_parents": { "type": ["array", "null"], @@ -5455,6 +5656,106 @@ "type": ["boolean", "null"], "description": "If true, automatically complete a review after every assigned reviewer fills all human review scores visible to them." }, + "user_id_fields": { + "type": ["array", "null"], + "items": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 2, + "maxItems": 32, + "description": "Path to the application's end-user identifier in metadata or span_attributes, for example [\"metadata\", \"customer_id\"]. This identifies an end user, not a session, request, organization, or the Braintrust user who created the project." + } + }, + "pattern_alert_destinations": { + "type": ["array", "null"], + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["webhook"], + "description": "The type of action to take" + }, + "url": { + "type": "string", + "description": "The webhook URL to send the request to" + }, + "formatting_prompt": { + "type": "string", + "minLength": 1, + "maxLength": 10000, + "description": "Instructions for Loop to format content sent to this destination" + } + }, + "required": ["type", "url"] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["slack"], + "description": "The type of action to take" + }, + "workspace_id": { + "type": "string", + "description": "The Slack workspace ID to post to" + }, + "channel": { + "type": "string", + "description": "The Slack channel ID to post to" + }, + "message_template": { + "type": "string", + "description": "Custom message template for the alert" + }, + "formatting_prompt": { + "type": "string", + "minLength": 1, + "maxLength": 10000, + "description": "Instructions for Loop to format content sent to this destination" + } + }, + "required": ["type", "workspace_id", "channel"] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["pagerduty"], + "description": "The type of action to take" + }, + "routing_key_secret_name": { + "type": "string", + "format": "uuid", + "description": "The data-plane secret containing the PagerDuty routing key" + }, + "severity": { + "type": "string", + "enum": ["critical", "error", "warning", "info"] + }, + "formatting_prompt": { + "type": "string", + "minLength": 1, + "maxLength": 10000, + "description": "Instructions for Loop to format content sent to this destination" + } + }, + "required": ["type", "routing_key_secret_name", "severity"] + } + ], + "description": "A delivery action available to an automation" + }, + "maxItems": 20, + "description": "Project-wide Slack, webhook, and PagerDuty destinations for new and reactivated Pattern alerts. An empty array disables alerts." + }, "default_preprocessor": { "$ref": "#/components/schemas/NullableSavedFunctionId" } @@ -5610,6 +5911,140 @@ }, "required": ["type", "content"], "title": "completion" + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["decisions"] + }, + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": ["system", "developer", "user", "assistant"] + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChatCompletionContentPartText" + } + } + ] + } + }, + "required": ["role", "content"] + } + } + ] + }, + "questions": { + "type": "array", + "items": { + "allOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "instructions": { + "type": "string" + } + }, + "required": ["instructions"] + }, + { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["predicate"] + } + }, + "required": ["type"] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["choice"] + }, + "choices": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "description": { + "type": "string" + } + }, + "required": ["value"] + } + } + }, + "required": ["type", "choices"] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["score"] + }, + "levels": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": ["label"] + } + } + }, + "required": ["type", "levels"] + } + ] + } + ] + } + } + }, + "required": ["type", "input", "questions"], + "title": "decisions" } ] }, @@ -5649,6 +6084,140 @@ "required": ["type", "content"], "title": "completion" }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["decisions"] + }, + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": ["system", "developer", "user", "assistant"] + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChatCompletionContentPartText" + } + } + ] + } + }, + "required": ["role", "content"] + } + } + ] + }, + "questions": { + "type": "array", + "items": { + "allOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "instructions": { + "type": "string" + } + }, + "required": ["instructions"] + }, + { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["predicate"] + } + }, + "required": ["type"] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["choice"] + }, + "choices": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ] + }, + "description": { + "type": "string" + } + }, + "required": ["value"] + } + } + }, + "required": ["type", "choices"] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["score"] + }, + "levels": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": ["label"] + } + } + }, + "required": ["type", "levels"] + } + ] + } + ] + } + } + }, + "required": ["type", "input", "questions"], + "title": "decisions" + }, { "type": "null" } @@ -6231,7 +6800,8 @@ }, "_internal_btql": { "type": ["object", "null"], - "additionalProperties": {} + "additionalProperties": {}, + "description": "Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer." } }, "required": ["dataset_id"], @@ -6257,7 +6827,8 @@ }, "_internal_btql": { "type": ["object", "null"], - "additionalProperties": {} + "additionalProperties": {}, + "description": "Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer." } }, "required": ["project_name", "dataset_name"], @@ -6270,6 +6841,10 @@ "data": { "type": "array", "items": {} + }, + "source_dataset_id": { + "type": "string", + "description": "The saved dataset these rows came from. Its project scopes saved simulated-user references; the caller must have read access to the dataset and source project." } }, "required": ["data"], @@ -6349,7 +6924,7 @@ }, "timeout": { "type": ["number", "null"], - "description": "The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout." + "description": "The maximum duration, in milliseconds, to run the evaluation. There is no timeout by default." }, "max_concurrency": { "type": ["number", "null"], @@ -6542,7 +7117,8 @@ "description": "Name of the service token" }, "preview_name": { - "type": "string" + "type": "string", + "description": "Masked preview showing the token prefix and last four characters, e.g. bt-st-…AbCd." }, "service_account_id": { "type": ["string", "null"], @@ -6666,6 +7242,7 @@ "preprocessor", "classifier", "review", + "decision", "question", "log" ], @@ -6967,7 +7544,7 @@ "type": "string", "minLength": 1, "maxLength": 10000, - "description": "Publish a Slack mrkdwn digest.\n\nPattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say \"No current pattern outcomes.\" This requirement overrides conflicting formatting guidance.\n\nInclude a complete \"*Pattern outcomes*\" section with one row for every included Pattern.\n\nUse this row format exactly:\n• — `outcome`\n\nIf a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern.\n\nAfter the outcome list, include a \"*Highlights*\" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report." + "description": "Instructions for Loop to format content sent to this destination" } }, "required": ["type", "workspace_id", "channel"], @@ -7264,6 +7841,7 @@ "scorers", "classifiers", "logs", + "patterns", "monitor", "for_review_project_log", "for_review_experiments", @@ -7355,6 +7933,10 @@ "options": { "type": "object", "properties": { + "filterScope": { + "type": "string", + "enum": ["span", "trace"] + }, "spanType": { "type": ["string", "null"], "enum": ["range", "frame"] @@ -7573,12 +8155,16 @@ }, "product_origin": { "type": ["string", "null"], - "enum": ["patterns"], + "enum": ["patterns", "dashboard", "digest"], "description": "The product surface that created and manages the automation" }, "status": { "$ref": "#/components/schemas/AutomationStatus" }, + "continuous": { + "type": "boolean", + "description": "Between scheduled runs, check for accumulated new work. The executor decides how to process it." + }, "threshold": { "type": "object", "properties": { @@ -7743,6 +8329,16 @@ "maxLength": 10000, "description": "Instructions for the Loop agent" }, + "digest": { + "type": "object", + "properties": { + "writing_style": { + "type": "string", + "enum": ["compact", "detailed"] + } + }, + "required": ["writing_style"] + }, "include_trigger_input": { "type": "boolean", "default": false, @@ -7764,13 +8360,13 @@ }, "harness": { "type": "string", - "enum": ["codex", "native", "claude-code"], - "description": "Required when saving a Loop automation. Currently only Codex is supported." + "enum": ["native", "codex", "claude-code", "pi"], + "description": "Required for an explicit model selection. Currently only Codex is supported." }, "model": { "type": "string", "minLength": 1, - "description": "Required when saving a Loop automation." + "description": "Omit harness, model, and endpoint_name to inherit the organization default provider and its recommended execution settings." }, "endpoint_name": { "type": "string", @@ -7787,6 +8383,21 @@ "xhigh", "max" ] + }, + "target": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["dashboard"] + }, + "id": { + "type": "string", + "enum": ["overview"] + } + }, + "required": ["type", "id"], + "description": "The object this automation writes to or maintains" } }, "required": ["prompt", "agent_slug"], @@ -7841,7 +8452,7 @@ "type": "string", "minLength": 1, "maxLength": 10000, - "description": "Publish a Slack mrkdwn digest.\n\nPattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say \"No current pattern outcomes.\" This requirement overrides conflicting formatting guidance.\n\nInclude a complete \"*Pattern outcomes*\" section with one row for every included Pattern.\n\nUse this row format exactly:\n• — `outcome`\n\nIf a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern.\n\nAfter the outcome list, include a \"*Highlights*\" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report." + "description": "Instructions for Loop to format content sent to this destination" } }, "required": ["type", "workspace_id", "channel"] diff --git a/js/src/generated_plain_types.ts b/js/src/generated_plain_types.ts index 708be55a2..7adc5cf44 100644 --- a/js/src/generated_plain_types.ts +++ b/js/src/generated_plain_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (content hash 387034529ea006a9) -- do not modify +// Auto-generated file (content hash 986b8a9b0d902f2d) -- do not modify export type AclObjectTypeType = /** @@ -39,7 +39,7 @@ export type PermissionType = /** * Each permission permits a certain type of operation on an object in the system -Permissions can be assigned to to objects on an individual basis, or grouped into roles +Permissions can be assigned to objects on an individual basis, or grouped into roles * * @enum create, read, update, delete, create_acls, read_acls, update_acls, delete_acls */ @@ -417,6 +417,9 @@ export type ApiKeyType = { * Name of the api key */ name: string; + /** + * Masked preview showing the key prefix and last four characters, e.g. sk-…AbCd. + */ preview_name: string; user_id?: | /** @@ -600,6 +603,7 @@ export type AsyncScoringControlType = */ kind: "score_update"; token?: string | undefined; + triggered_xact_id?: string | undefined; } | { /** @@ -1382,7 +1386,7 @@ export type CodeBundleType = { */ provider: "modal"; /** - * sandbox snapshot ref + * The Modal snapshot to start the sandbox from */ snapshot_ref: string; } @@ -1673,7 +1677,7 @@ export type DatasetEventType = { ) | undefined; /** - * A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing + * A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing */ span_id: string; /** @@ -1767,7 +1771,7 @@ export type DatasetSnapshotType = { name: string; description: string | null; /** - * Transaction id of the brainstore version at the time of the snapshot + * Transaction ID of the dataset version at the time of the snapshot */ xact_id: string; /** @@ -1900,7 +1904,7 @@ export type EnvVarType = { ) | undefined; /** - * The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys + * The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys, sandbox_provider for sandbox provider credentials, automation_integration for automation delivery credentials * * @default "env_var" * @enum env_var, ai_provider, sandbox_provider, automation_integration @@ -2244,12 +2248,12 @@ export type SpanTypeType = /** * Type of the span, for display purposes only * - * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, question, log + * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, decision, question, log */ | /** * Type of the span, for display purposes only * - * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, question, log + * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, decision, question, log */ (| "llm" | "score" @@ -2262,13 +2266,14 @@ export type SpanTypeType = | "preprocessor" | "classifier" | "review" + | "decision" | "question" | "log" ) /** * Type of the span, for display purposes only * - * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, question, log + * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, decision, question, log */ | null; export type SpanAttributesType = @@ -2389,14 +2394,14 @@ export type ExperimentEventType = { unknown | undefined; scores?: | /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments. */ /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments. */ (| {} /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments. */ | null ) @@ -2514,7 +2519,7 @@ export type ExperimentEventType = { ) | undefined; /** - * A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing + * A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing */ span_id: string; span_parents?: @@ -2675,6 +2680,54 @@ export type PromptBlockDataNullishType = type: "completion"; content: string; } + | { + /** + * @enum decisions + */ + type: "decisions"; + input: + | string + | Array<{ + /** + * @enum system, developer, user, assistant + */ + role: "system" | "developer" | "user" | "assistant"; + content: string | Array; + }>; + questions: Array< + { + name?: string | undefined; + instructions: string; + } & ( + | { + /** + * @enum predicate + */ + type: "predicate"; + } + | { + /** + * @enum choice + */ + type: "choice"; + choices: Array<{ + value: string | boolean; + description?: string | undefined; + }>; + } + | { + /** + * @enum score + */ + type: "score"; + levels: Array<{ + label: string; + description?: string | undefined; + }>; + } + ) + >; + } | null; export type ModelParamsType = | Partial< @@ -2946,6 +2999,54 @@ export type PromptBlockDataType = */ type: "completion"; content: string; + } + | { + /** + * @enum decisions + */ + type: "decisions"; + input: + | string + | Array<{ + /** + * @enum system, developer, user, assistant + */ + role: "system" | "developer" | "user" | "assistant"; + content: string | Array; + }>; + questions: Array< + { + name?: string | undefined; + instructions: string; + } & ( + | { + /** + * @enum predicate + */ + type: "predicate"; + } + | { + /** + * @enum choice + */ + type: "choice"; + choices: Array<{ + value: string | boolean; + description?: string | undefined; + }>; + } + | { + /** + * @enum score + */ + type: "score"; + levels: Array<{ + label: string; + description?: string | undefined; + }>; + } + ) + >; }; export type GraphNodeType = | { @@ -3465,7 +3566,149 @@ export type FunctionDataType = additionalProperties?: boolean | undefined; }; } - | (TopicMapDataType & unknown); + | (TopicMapDataType & unknown) + /** + * A task function that runs a conversation between an agent and a simulated user. + */ + | { + /** + * @enum simulation + */ + type: "simulation"; + config: + | { + /** + * Who speaks first in the conversation + * + * @default "agent" + * @enum agent, simulated_user + */ + initiator: "agent" | "simulated_user"; + /** + * Maximum conversation duration in seconds + * + * @default 600 + * @minimum 15 + * @maximum 3600 + */ + timeout_seconds: number; + /** + * @default "voice" + * @enum voice + */ + modality: "voice"; + agent: + | { + /** + * @enum livekit + */ + type: "livekit"; + /** + * The WebSocket URL of the LiveKit project hosting the agent + * + * @minLength 1 + */ + server_url: string; + /** + * The name of the LiveKit agent to dispatch + * + * @minLength 1 + */ + agent_name: string; + } + | { + /** + * @enum phone + */ + type: "phone"; + /** + * @pattern ^\+[1-9]\d{7,14}$ + */ + phone_number: string; + } + | { + /** + * @enum sip + */ + type: "sip"; + /** + * @pattern ^sips?:[^\s]+$ + */ + uri: string; + }; + } + | { + /** + * Who speaks first in the conversation + * + * @default "agent" + * @enum agent, simulated_user + */ + initiator: "agent" | "simulated_user"; + /** + * Maximum conversation duration in seconds + * + * @default 600 + * @minimum 15 + * @maximum 3600 + */ + timeout_seconds: number; + /** + * @enum text + */ + modality: "text"; + agent: + | { + /** + * @enum livekit + */ + type: "livekit"; + /** + * The WebSocket URL of the LiveKit project hosting the agent + * + * @minLength 1 + */ + server_url: string; + /** + * The name of the LiveKit agent to dispatch + * + * @minLength 1 + */ + agent_name: string; + } + | { + /** + * @enum acp + */ + type: "acp"; + /** + * Executable for the ACP agent subprocess, communicating over stdio + * + * @minLength 1 + */ + command: string; + /** + * Arguments passed to the executable without shell interpretation + * + * @default [] + */ + args: Array; + /** + * Absolute working directory for the ACP session on the execution host + * + * @pattern ^(?:[/]|[A-Za-z]:[\\/]|\\\\) + */ + cwd: string; + /** + * Deny ACP permission requests; this does not sandbox the subprocess + * + * @default "deny" + * @enum deny + */ + permissions: "deny"; + }; + }; + }; export type FunctionType = { /** * Unique identifier for the prompt @@ -3565,14 +3808,14 @@ export type FunctionType = { object_id: string; internal?: | /** - * The function exists for internal purposes and should not be displayed in the list of functions. + * If true, the function was created for use by another Braintrust object and is hidden from function lists in the Braintrust UI. */ /** - * The function exists for internal purposes and should not be displayed in the list of functions. + * If true, the function was created for use by another Braintrust object and is hidden from function lists in the Braintrust UI. */ (| boolean /** - * The function exists for internal purposes and should not be displayed in the list of functions. + * If true, the function was created for use by another Braintrust object and is hidden from function lists in the Braintrust UI. */ | null ) @@ -3599,9 +3842,9 @@ export type FunctionType = { }; export type FunctionFormatType = /** - * @enum llm, code, global, graph, topic_map + * @enum llm, code, global, graph, topic_map, simulation */ - "llm" | "code" | "global" | "graph" | "topic_map"; + "llm" | "code" | "global" | "graph" | "topic_map" | "simulation"; export type PromptDataType = Partial<{ prompt: PromptBlockDataNullishType; options: PromptOptionsNullishType; @@ -3988,10 +4231,10 @@ export type InvokeParentType = object_id: string; row_ids?: | /** - * Identifiers for the row to to log a subspan under + * Identifiers for the row to log a subspan under */ /** - * Identifiers for the row to to log a subspan under + * Identifiers for the row to log a subspan under */ (| { /** @@ -4008,7 +4251,7 @@ export type InvokeParentType = root_span_id: string; } /** - * Identifiers for the row to to log a subspan under + * Identifiers for the row to log a subspan under */ | null ) @@ -4056,6 +4299,10 @@ export type InvokeFunctionType = */ FunctionIdType & Partial<{ + /** + * A label identifying the caller, used to attribute usage + */ + request_source: string; /** * Argument to the function, which can be any JSON serializable value */ @@ -4065,14 +4312,14 @@ export type InvokeFunctionType = */ expected: unknown; /** - * Any relevant metadata. This will be logged and available as the `metadata` argument. + * Any relevant metadata to log on the span. For prompts, this is available as the `metadata` template variable. For code functions, this is passed through hooks and is not added to `input`. */ metadata: /** - * Any relevant metadata. This will be logged and available as the `metadata` argument. + * Any relevant metadata to log on the span. For prompts, this is available as the `metadata` template variable. For code functions, this is passed through hooks and is not added to `input`. */ | {} /** - * Any relevant metadata. This will be logged and available as the `metadata` argument. + * Any relevant metadata to log on the span. For prompts, this is available as the `metadata` template variable. For code functions, this is passed through hooks and is not added to `input`. */ | null; /** @@ -4323,6 +4570,14 @@ export type TraceScopeType = { }; export type OnlineScoreConfigType = { status?: AutomationStatusType | undefined; + service_tier?: + | /** + * Service tier for online scoring. If omitted, prompt-level service tiers are preserved. + * + * @enum standard, flex + */ + ("standard" | "flex") + | undefined; /** * The sampling rate for online scoring * @@ -4406,11 +4661,76 @@ export type OrganizationType = { * Name of the organization */ name: string; - api_url?: (string | null) | undefined; - is_universal_api?: (boolean | null) | undefined; - is_dataplane_private?: (boolean | null) | undefined; - proxy_url?: (string | null) | undefined; - realtime_url?: (string | null) | undefined; + api_url?: + | /** + * URL of the organization's data plane API. If unset, the Braintrust-hosted API is used. + */ + /** + * URL of the organization's data plane API. If unset, the Braintrust-hosted API is used. + */ + (| string + /** + * URL of the organization's data plane API. If unset, the Braintrust-hosted API is used. + */ + | null + ) + | undefined; + is_universal_api?: + | /** + * Whether the data plane at `api_url` also serves the AI proxy, at `api_url` with `/v1/proxy` appended + */ + /** + * Whether the data plane at `api_url` also serves the AI proxy, at `api_url` with `/v1/proxy` appended + */ + (| boolean + /** + * Whether the data plane at `api_url` also serves the AI proxy, at `api_url` with `/v1/proxy` appended + */ + | null + ) + | undefined; + is_dataplane_private?: + | /** + * Whether the organization's data plane is only reachable from a private network + */ + /** + * Whether the organization's data plane is only reachable from a private network + */ + (| boolean + /** + * Whether the organization's data plane is only reachable from a private network + */ + | null + ) + | undefined; + proxy_url?: + | /** + * URL of the organization's AI proxy + */ + /** + * URL of the organization's AI proxy + */ + (| string + /** + * URL of the organization's AI proxy + */ + | null + ) + | undefined; + realtime_url?: + | /** + * URL of the organization's realtime service + */ + /** + * URL of the organization's realtime service + */ + (| string + /** + * URL of the organization's realtime service + */ + | null + ) + | undefined; created?: | /** * Date of organization creation @@ -4425,8 +4745,34 @@ export type OrganizationType = { | null ) | undefined; - archived_at?: (string | null) | undefined; - deleted_at?: (string | null) | undefined; + archived_at?: + | /** + * Date the organization was archived + */ + /** + * Date the organization was archived + */ + (| string + /** + * Date the organization was archived + */ + | null + ) + | undefined; + deleted_at?: + | /** + * Date the organization was deleted + */ + /** + * Date the organization was deleted + */ + (| string + /** + * Date the organization was deleted + */ + | null + ) + | undefined; image_rendering_mode?: ImageRenderingModeType | undefined; }; export type RetentionObjectTypeType = @@ -4631,6 +4977,100 @@ export type ProjectSettingsType = Partial<{ * If true, automatically complete a review after every assigned reviewer fills all human review scores visible to them. */ | null; + user_id_fields: Array< + /** + * Path to the application's end-user identifier in metadata or span_attributes, for example ["metadata", "customer_id"]. This identifies an end user, not a session, request, organization, or the Braintrust user who created the project. + */ + Array + > | null; + /** + * Project-wide Slack, webhook, and PagerDuty destinations for new and reactivated Pattern alerts. An empty array disables alerts. + */ + pattern_alert_destinations: /** + * Project-wide Slack, webhook, and PagerDuty destinations for new and reactivated Pattern alerts. An empty array disables alerts. + */ + | Array< + /** + * A delivery action available to an automation + */ + | { + /** + * The type of action to take + * + * @enum webhook + */ + type: "webhook"; + /** + * The webhook URL to send the request to + */ + url: string; + formatting_prompt?: /** + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ + string | undefined; + } + | { + /** + * The type of action to take + * + * @enum slack + */ + type: "slack"; + /** + * The Slack workspace ID to post to + */ + workspace_id: string; + /** + * The Slack channel ID to post to + */ + channel: string; + message_template?: /** + * Custom message template for the alert + */ + string | undefined; + formatting_prompt?: /** + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ + string | undefined; + } + | { + /** + * The type of action to take + * + * @enum pagerduty + */ + type: "pagerduty"; + /** + * The data-plane secret containing the PagerDuty routing key + */ + routing_key_secret_name: string; + /** + * @enum critical, error, warning, info + */ + severity: "critical" | "error" | "warning" | "info"; + formatting_prompt?: /** + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ + string | undefined; + } + > + /** + * Project-wide Slack, webhook, and PagerDuty destinations for new and reactivated Pattern alerts. An empty array disables alerts. + */ + | null; default_preprocessor: NullableSavedFunctionIdType; }> | null; export type ProjectType = { @@ -4715,23 +5155,27 @@ export type WindowedAutomationConfigType = { | /** * The product surface that created and manages the automation * - * @enum patterns + * @enum patterns, dashboard, digest */ (| /** * The product surface that created and manages the automation * - * @enum patterns + * @enum patterns, dashboard, digest */ - "patterns" + ("patterns" | "dashboard" | "digest") /** * The product surface that created and manages the automation * - * @enum patterns + * @enum patterns, dashboard, digest */ | null ) | undefined; status?: AutomationStatusType | undefined; + continuous?: /** + * Between scheduled runs, check for accumulated new work. The executor decides how to process it. + */ + boolean | undefined; threshold?: /** * Optional calculation and lifecycle policy that gate scheduled delivery */ @@ -4899,6 +5343,14 @@ export type WindowedAutomationConfigType = { * @maxLength 10000 */ prompt: string; + digest?: + | { + /** + * @enum compact, detailed + */ + writing_style: "compact" | "detailed"; + } + | undefined; /** * Whether to include the automation trigger payload as input * @@ -4922,14 +5374,14 @@ export type WindowedAutomationConfigType = { string>; harness?: | /** - * Required when saving a Loop automation. Currently only Codex is supported. + * Required for an explicit model selection. Currently only Codex is supported. * - * @enum codex, native, claude-code + * @enum native, codex, claude-code, pi */ - ("codex" | "native" | "claude-code") + ("native" | "codex" | "claude-code" | "pi") | undefined; model?: /** - * Required when saving a Loop automation. + * Omit harness, model, and endpoint_name to inherit the organization default provider and its recommended execution settings. * * @minLength 1 */ @@ -4944,6 +5396,20 @@ export type WindowedAutomationConfigType = { */ ("none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max") | undefined; + target?: /** + * The object this automation writes to or maintains + */ + | { + /** + * @enum dashboard + */ + type: "dashboard"; + /** + * @enum overview + */ + id: "overview"; + } + | undefined; } | undefined; /** @@ -4994,22 +5460,11 @@ export type WindowedAutomationConfigType = { */ string | undefined; formatting_prompt?: /** - * Publish a Slack mrkdwn digest. - - Pattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say "No current pattern outcomes." This requirement overrides conflicting formatting guidance. - - Include a complete "*Pattern outcomes*" section with one row for every included Pattern. - - Use this row format exactly: - • — `outcome` - - If a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern. - - After the outcome list, include a "*Highlights*" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report. - * - * @minLength 1 - * @maxLength 10000 - */ + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ string | undefined; } | { @@ -5243,22 +5698,11 @@ export type TopicDigestAutomationConfigType = { */ string | undefined; formatting_prompt?: /** - * Publish a Slack mrkdwn digest. - - Pattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say "No current pattern outcomes." This requirement overrides conflicting formatting guidance. - - Include a complete "*Pattern outcomes*" section with one row for every included Pattern. - - Use this row format exactly: - • — `outcome` - - If a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern. - - After the outcome list, include a "*Highlights*" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report. - * - * @minLength 1 - * @maxLength 10000 - */ + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ string | undefined; }; topic_map_function_ids?: /** @@ -5387,22 +5831,11 @@ export type ProjectAutomationType = { */ string | undefined; formatting_prompt?: /** - * Publish a Slack mrkdwn digest. - - Pattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say "No current pattern outcomes." This requirement overrides conflicting formatting guidance. - - Include a complete "*Pattern outcomes*" section with one row for every included Pattern. - - Use this row format exactly: - • — `outcome` - - If a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern. - - After the outcome list, include a "*Highlights*" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report. - * - * @minLength 1 - * @maxLength 10000 - */ + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ string | undefined; } | { @@ -5659,22 +6092,11 @@ export type ProjectAutomationType = { */ string | undefined; formatting_prompt?: /** - * Publish a Slack mrkdwn digest. - - Pattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say "No current pattern outcomes." This requirement overrides conflicting formatting guidance. - - Include a complete "*Pattern outcomes*" section with one row for every included Pattern. - - Use this row format exactly: - • — `outcome` - - If a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern. - - After the outcome list, include a "*Highlights*" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report. - * - * @minLength 1 - * @maxLength 10000 - */ + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ string | undefined; } | { @@ -5783,7 +6205,7 @@ export type ProjectGroupType = { ) | undefined; /** - * Sorted ids of active projects in this project group + * Sorted ids of active projects in this project group. Modifying membership requires project_group:update on the group. */ member_projects: Array; }; @@ -5846,14 +6268,14 @@ export type ProjectLogsEventType = { unknown | undefined; scores?: | /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. */ /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. */ (| {} /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. */ | null ) @@ -5971,7 +6393,7 @@ export type ProjectLogsEventType = { ) | undefined; /** - * A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing + * A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing */ span_id: string; span_parents?: @@ -6620,7 +7042,20 @@ export type RunEvalType = { | null ) | undefined; - _internal_btql?: ({} | null) | undefined; + _internal_btql?: + | /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + (| {} + /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + | null + ) + | undefined; } /** * Project and dataset name @@ -6656,13 +7091,30 @@ export type RunEvalType = { | null ) | undefined; - _internal_btql?: ({} | null) | undefined; + _internal_btql?: + | /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + (| {} + /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + | null + ) + | undefined; } /** * Dataset rows */ | { data: Array; + source_dataset_id?: /** + * The saved dataset these rows came from. Its project scopes saved simulated-user references; the caller must have read access to the dataset and source project. + */ + string | undefined; } /** * Experiment whose inputs and outputs should be used as dataset inputs and expected values @@ -6734,14 +7186,14 @@ export type RunEvalType = { | undefined; timeout?: | /** - * The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout. + * The maximum duration, in milliseconds, to run the evaluation. There is no timeout by default. */ /** - * The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout. + * The maximum duration, in milliseconds, to run the evaluation. There is no timeout by default. */ (| number /** - * The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout. + * The maximum duration, in milliseconds, to run the evaluation. There is no timeout by default. */ | null ) @@ -6914,6 +7366,9 @@ export type ServiceTokenType = { * Name of the service token */ name: string; + /** + * Masked preview showing the token prefix and last four characters, e.g. bt-st-…AbCd. + */ preview_name: string; service_account_id?: | /** @@ -7229,6 +7684,10 @@ export type ViewOptionsType = */ viewType: "monitor"; options: Partial<{ + /** + * @enum span, trace + */ + filterScope: "span" | "trace"; /** * @enum range, frame */ @@ -7359,7 +7818,7 @@ export type ViewType = { /** * Type of object that the view corresponds to. * - * @enum projects, experiments, experiment, playgrounds, playground, datasets, dataset, prompts, parameters, tools, scorers, classifiers, logs, monitor, for_review_project_log, for_review_experiments, for_review_datasets + * @enum projects, experiments, experiment, playgrounds, playground, datasets, dataset, prompts, parameters, tools, scorers, classifiers, logs, patterns, monitor, for_review_project_log, for_review_experiments, for_review_datasets */ view_type: | "projects" @@ -7375,6 +7834,7 @@ export type ViewType = { | "scorers" | "classifiers" | "logs" + | "patterns" | "monitor" | "for_review_project_log" | "for_review_experiments" diff --git a/js/src/generated_types.ts b/js/src/generated_types.ts index ea2bc045b..7c1aff8e5 100644 --- a/js/src/generated_types.ts +++ b/js/src/generated_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (content hash 387034529ea006a9) -- do not modify +// Auto-generated file (content hash 986b8a9b0d902f2d) -- do not modify import { z } from "zod/v3"; @@ -191,7 +191,11 @@ export const AsyncScoringState = z.union([ ]); export type AsyncScoringStateType = z.infer; export const AsyncScoringControl = z.union([ - z.object({ kind: z.literal("score_update"), token: z.string().optional() }), + z.object({ + kind: z.literal("score_update"), + token: z.string().optional(), + triggered_xact_id: z.string().optional(), + }), z.object({ kind: z.literal("state_override"), state: AsyncScoringState }), z.object({ kind: z.literal("state_force_reselect") }), z.object({ kind: z.literal("state_enabled_force_rescore") }), @@ -829,6 +833,7 @@ export const SpanType = z.union([ "preprocessor", "classifier", "review", + "decision", "question", "log", ]), @@ -970,6 +975,46 @@ export const PromptBlockDataNullish = z.union([ tools: z.string().optional(), }), z.object({ type: z.literal("completion"), content: z.string() }), + z.object({ + type: z.literal("decisions"), + input: z.union([ + z.string(), + z.array( + z.object({ + role: z.enum(["system", "developer", "user", "assistant"]), + content: z.union([ + z.string(), + z.array(ChatCompletionContentPartText), + ]), + }), + ), + ]), + questions: z.array( + z.object({ name: z.string().optional(), instructions: z.string() }).and( + z.union([ + z.object({ type: z.literal("predicate") }), + z.object({ + type: z.literal("choice"), + choices: z.array( + z.object({ + value: z.union([z.string(), z.boolean()]), + description: z.string().optional(), + }), + ), + }), + z.object({ + type: z.literal("score"), + levels: z.array( + z.object({ + label: z.string(), + description: z.string().optional(), + }), + ), + }), + ]), + ), + ), + }), z.null(), ]); export type PromptBlockDataNullishType = z.infer; @@ -1173,6 +1218,46 @@ export const PromptBlockData = z.union([ tools: z.string().optional(), }), z.object({ type: z.literal("completion"), content: z.string() }), + z.object({ + type: z.literal("decisions"), + input: z.union([ + z.string(), + z.array( + z.object({ + role: z.enum(["system", "developer", "user", "assistant"]), + content: z.union([ + z.string(), + z.array(ChatCompletionContentPartText), + ]), + }), + ), + ]), + questions: z.array( + z.object({ name: z.string().optional(), instructions: z.string() }).and( + z.union([ + z.object({ type: z.literal("predicate") }), + z.object({ + type: z.literal("choice"), + choices: z.array( + z.object({ + value: z.union([z.string(), z.boolean()]), + description: z.string().optional(), + }), + ), + }), + z.object({ + type: z.literal("score"), + levels: z.array( + z.object({ + label: z.string(), + description: z.string().optional(), + }), + ), + }), + ]), + ), + ), + }), ]); export type PromptBlockDataType = z.infer; export const GraphNode = z.union([ @@ -1297,6 +1382,68 @@ export const FunctionData = z.union([ }), }), TopicMapData.and(z.unknown()), + z.object({ + type: z.literal("simulation"), + config: z.union([ + z.object({ + initiator: z + .enum(["agent", "simulated_user"]) + .optional() + .default("agent"), + timeout_seconds: z + .number() + .int() + .gte(15) + .lte(3600) + .optional() + .default(600), + modality: z.literal("voice").optional().default("voice"), + agent: z.union([ + z.object({ + type: z.literal("livekit"), + server_url: z.string().min(1), + agent_name: z.string().min(1), + }), + z.object({ + type: z.literal("phone"), + phone_number: z.string().regex(/^\+[1-9]\d{7,14}$/), + }), + z.object({ + type: z.literal("sip"), + uri: z.string().regex(/^sips?:[^\s]+$/), + }), + ]), + }), + z.object({ + initiator: z + .enum(["agent", "simulated_user"]) + .optional() + .default("agent"), + timeout_seconds: z + .number() + .int() + .gte(15) + .lte(3600) + .optional() + .default(600), + modality: z.literal("text"), + agent: z.union([ + z.object({ + type: z.literal("livekit"), + server_url: z.string().min(1), + agent_name: z.string().min(1), + }), + z.object({ + type: z.literal("acp"), + command: z.string().min(1), + args: z.array(z.string()).optional().default([]), + cwd: z.string().regex(/^(?:[\/]|[A-Za-z]:[\\\/]|\\\\)/), + permissions: z.literal("deny").optional().default("deny"), + }), + ]), + }), + ]), + }), ]); export type FunctionDataType = z.infer; export const Function = z.object({ @@ -1340,6 +1487,7 @@ export const FunctionFormat = z.enum([ "global", "graph", "topic_map", + "simulation", ]); export type FunctionFormatType = z.infer; export const PromptData = z @@ -1521,6 +1669,7 @@ export type StreamingModeType = z.infer; export const InvokeFunction = FunctionId.and( z .object({ + request_source: z.string(), input: z.unknown(), expected: z.unknown(), metadata: z.union([z.object({}).partial().passthrough(), z.null()]), @@ -1605,6 +1754,7 @@ export type TraceScopeType = z.infer; export const OnlineScoreConfig = z.union([ z.object({ status: AutomationStatus.optional(), + service_tier: z.enum(["standard", "flex"]).optional(), sampling_rate: z.number().gte(0).lte(1), scorers: z.array(SavedFunctionId), btql_filter: z.union([z.string(), z.null()]).optional(), @@ -1683,6 +1833,35 @@ export const ProjectSettings = z.union([ coding_agent_insights_dashboard: z.union([z.boolean(), z.null()]), blind_reviews: z.union([z.boolean(), z.null()]), require_all_human_review_scores: z.union([z.boolean(), z.null()]), + user_id_fields: z.union([ + z.array(z.array(z.string().min(1).max(256)).min(2).max(32)), + z.null(), + ]), + pattern_alert_destinations: z.union([ + z.array( + z.union([ + z.object({ + type: z.literal("webhook"), + url: z.string(), + formatting_prompt: z.string().min(1).max(10000).optional(), + }), + z.object({ + type: z.literal("slack"), + workspace_id: z.string(), + channel: z.string(), + message_template: z.string().optional(), + formatting_prompt: z.string().min(1).max(10000).optional(), + }), + z.object({ + type: z.literal("pagerduty"), + routing_key_secret_name: z.string().uuid(), + severity: z.enum(["critical", "error", "warning", "info"]), + formatting_prompt: z.string().min(1).max(10000).optional(), + }), + ]), + ), + z.null(), + ]), default_preprocessor: NullableSavedFunctionId, }) .partial(), @@ -1702,8 +1881,11 @@ export const Project = z.object({ export type ProjectType = z.infer; export const WindowedAutomationConfig = z.object({ event_type: z.literal("windowed"), - product_origin: z.union([z.literal("patterns"), z.null()]).optional(), + product_origin: z + .union([z.enum(["patterns", "dashboard", "digest"]), z.null()]) + .optional(), status: AutomationStatus.optional(), + continuous: z.boolean().optional(), threshold: z .object({ calculation: z.object({ @@ -1745,15 +1927,21 @@ export const WindowedAutomationConfig = z.object({ loop: z .object({ prompt: z.string().min(1).max(10000), + digest: z + .object({ writing_style: z.enum(["compact", "detailed"]) }) + .optional(), include_trigger_input: z.boolean().optional().default(false), agent_slug: z.string().min(1), auto_approve_tools: z.array(z.string().min(1)).optional().default([]), - harness: z.enum(["codex", "native", "claude-code"]).optional(), + harness: z.enum(["native", "codex", "claude-code", "pi"]).optional(), model: z.string().min(1).optional(), endpoint_name: z.string().min(1).optional(), reasoning_effort: z .enum(["none", "minimal", "low", "medium", "high", "xhigh", "max"]) .optional(), + target: z + .object({ type: z.literal("dashboard"), id: z.literal("overview") }) + .optional(), }) .optional(), actions: z @@ -2240,7 +2428,10 @@ export const RunEval = z.object({ .union([z.object({}).partial().passthrough(), z.null()]) .optional(), }), - z.object({ data: z.array(z.unknown()) }), + z.object({ + data: z.array(z.unknown()), + source_dataset_id: z.string().optional(), + }), z.object({ experiment_name: z.string() }), ]), name: z.string().optional(), @@ -2375,6 +2566,7 @@ export const ViewOptions = z.union([ viewType: z.literal("monitor"), options: z .object({ + filterScope: z.enum(["span", "trace"]), spanType: z.union([z.enum(["range", "frame"]), z.null()]), rangeValue: z.union([z.string(), z.null()]), frameStart: z.union([z.string(), z.null()]), @@ -2475,6 +2667,7 @@ export const View = z.object({ "scorers", "classifiers", "logs", + "patterns", "monitor", "for_review_project_log", "for_review_experiments", diff --git a/js/util/generated_plain_types.ts b/js/util/generated_plain_types.ts index 708be55a2..7adc5cf44 100644 --- a/js/util/generated_plain_types.ts +++ b/js/util/generated_plain_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (content hash 387034529ea006a9) -- do not modify +// Auto-generated file (content hash 986b8a9b0d902f2d) -- do not modify export type AclObjectTypeType = /** @@ -39,7 +39,7 @@ export type PermissionType = /** * Each permission permits a certain type of operation on an object in the system -Permissions can be assigned to to objects on an individual basis, or grouped into roles +Permissions can be assigned to objects on an individual basis, or grouped into roles * * @enum create, read, update, delete, create_acls, read_acls, update_acls, delete_acls */ @@ -417,6 +417,9 @@ export type ApiKeyType = { * Name of the api key */ name: string; + /** + * Masked preview showing the key prefix and last four characters, e.g. sk-…AbCd. + */ preview_name: string; user_id?: | /** @@ -600,6 +603,7 @@ export type AsyncScoringControlType = */ kind: "score_update"; token?: string | undefined; + triggered_xact_id?: string | undefined; } | { /** @@ -1382,7 +1386,7 @@ export type CodeBundleType = { */ provider: "modal"; /** - * sandbox snapshot ref + * The Modal snapshot to start the sandbox from */ snapshot_ref: string; } @@ -1673,7 +1677,7 @@ export type DatasetEventType = { ) | undefined; /** - * A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing + * A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing */ span_id: string; /** @@ -1767,7 +1771,7 @@ export type DatasetSnapshotType = { name: string; description: string | null; /** - * Transaction id of the brainstore version at the time of the snapshot + * Transaction ID of the dataset version at the time of the snapshot */ xact_id: string; /** @@ -1900,7 +1904,7 @@ export type EnvVarType = { ) | undefined; /** - * The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys + * The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys, sandbox_provider for sandbox provider credentials, automation_integration for automation delivery credentials * * @default "env_var" * @enum env_var, ai_provider, sandbox_provider, automation_integration @@ -2244,12 +2248,12 @@ export type SpanTypeType = /** * Type of the span, for display purposes only * - * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, question, log + * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, decision, question, log */ | /** * Type of the span, for display purposes only * - * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, question, log + * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, decision, question, log */ (| "llm" | "score" @@ -2262,13 +2266,14 @@ export type SpanTypeType = | "preprocessor" | "classifier" | "review" + | "decision" | "question" | "log" ) /** * Type of the span, for display purposes only * - * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, question, log + * @enum llm, score, function, eval, task, tool, automation, facet, preprocessor, classifier, review, decision, question, log */ | null; export type SpanAttributesType = @@ -2389,14 +2394,14 @@ export type ExperimentEventType = { unknown | undefined; scores?: | /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments. */ /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments. */ (| {} /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments. */ | null ) @@ -2514,7 +2519,7 @@ export type ExperimentEventType = { ) | undefined; /** - * A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing + * A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing */ span_id: string; span_parents?: @@ -2675,6 +2680,54 @@ export type PromptBlockDataNullishType = type: "completion"; content: string; } + | { + /** + * @enum decisions + */ + type: "decisions"; + input: + | string + | Array<{ + /** + * @enum system, developer, user, assistant + */ + role: "system" | "developer" | "user" | "assistant"; + content: string | Array; + }>; + questions: Array< + { + name?: string | undefined; + instructions: string; + } & ( + | { + /** + * @enum predicate + */ + type: "predicate"; + } + | { + /** + * @enum choice + */ + type: "choice"; + choices: Array<{ + value: string | boolean; + description?: string | undefined; + }>; + } + | { + /** + * @enum score + */ + type: "score"; + levels: Array<{ + label: string; + description?: string | undefined; + }>; + } + ) + >; + } | null; export type ModelParamsType = | Partial< @@ -2946,6 +2999,54 @@ export type PromptBlockDataType = */ type: "completion"; content: string; + } + | { + /** + * @enum decisions + */ + type: "decisions"; + input: + | string + | Array<{ + /** + * @enum system, developer, user, assistant + */ + role: "system" | "developer" | "user" | "assistant"; + content: string | Array; + }>; + questions: Array< + { + name?: string | undefined; + instructions: string; + } & ( + | { + /** + * @enum predicate + */ + type: "predicate"; + } + | { + /** + * @enum choice + */ + type: "choice"; + choices: Array<{ + value: string | boolean; + description?: string | undefined; + }>; + } + | { + /** + * @enum score + */ + type: "score"; + levels: Array<{ + label: string; + description?: string | undefined; + }>; + } + ) + >; }; export type GraphNodeType = | { @@ -3465,7 +3566,149 @@ export type FunctionDataType = additionalProperties?: boolean | undefined; }; } - | (TopicMapDataType & unknown); + | (TopicMapDataType & unknown) + /** + * A task function that runs a conversation between an agent and a simulated user. + */ + | { + /** + * @enum simulation + */ + type: "simulation"; + config: + | { + /** + * Who speaks first in the conversation + * + * @default "agent" + * @enum agent, simulated_user + */ + initiator: "agent" | "simulated_user"; + /** + * Maximum conversation duration in seconds + * + * @default 600 + * @minimum 15 + * @maximum 3600 + */ + timeout_seconds: number; + /** + * @default "voice" + * @enum voice + */ + modality: "voice"; + agent: + | { + /** + * @enum livekit + */ + type: "livekit"; + /** + * The WebSocket URL of the LiveKit project hosting the agent + * + * @minLength 1 + */ + server_url: string; + /** + * The name of the LiveKit agent to dispatch + * + * @minLength 1 + */ + agent_name: string; + } + | { + /** + * @enum phone + */ + type: "phone"; + /** + * @pattern ^\+[1-9]\d{7,14}$ + */ + phone_number: string; + } + | { + /** + * @enum sip + */ + type: "sip"; + /** + * @pattern ^sips?:[^\s]+$ + */ + uri: string; + }; + } + | { + /** + * Who speaks first in the conversation + * + * @default "agent" + * @enum agent, simulated_user + */ + initiator: "agent" | "simulated_user"; + /** + * Maximum conversation duration in seconds + * + * @default 600 + * @minimum 15 + * @maximum 3600 + */ + timeout_seconds: number; + /** + * @enum text + */ + modality: "text"; + agent: + | { + /** + * @enum livekit + */ + type: "livekit"; + /** + * The WebSocket URL of the LiveKit project hosting the agent + * + * @minLength 1 + */ + server_url: string; + /** + * The name of the LiveKit agent to dispatch + * + * @minLength 1 + */ + agent_name: string; + } + | { + /** + * @enum acp + */ + type: "acp"; + /** + * Executable for the ACP agent subprocess, communicating over stdio + * + * @minLength 1 + */ + command: string; + /** + * Arguments passed to the executable without shell interpretation + * + * @default [] + */ + args: Array; + /** + * Absolute working directory for the ACP session on the execution host + * + * @pattern ^(?:[/]|[A-Za-z]:[\\/]|\\\\) + */ + cwd: string; + /** + * Deny ACP permission requests; this does not sandbox the subprocess + * + * @default "deny" + * @enum deny + */ + permissions: "deny"; + }; + }; + }; export type FunctionType = { /** * Unique identifier for the prompt @@ -3565,14 +3808,14 @@ export type FunctionType = { object_id: string; internal?: | /** - * The function exists for internal purposes and should not be displayed in the list of functions. + * If true, the function was created for use by another Braintrust object and is hidden from function lists in the Braintrust UI. */ /** - * The function exists for internal purposes and should not be displayed in the list of functions. + * If true, the function was created for use by another Braintrust object and is hidden from function lists in the Braintrust UI. */ (| boolean /** - * The function exists for internal purposes and should not be displayed in the list of functions. + * If true, the function was created for use by another Braintrust object and is hidden from function lists in the Braintrust UI. */ | null ) @@ -3599,9 +3842,9 @@ export type FunctionType = { }; export type FunctionFormatType = /** - * @enum llm, code, global, graph, topic_map + * @enum llm, code, global, graph, topic_map, simulation */ - "llm" | "code" | "global" | "graph" | "topic_map"; + "llm" | "code" | "global" | "graph" | "topic_map" | "simulation"; export type PromptDataType = Partial<{ prompt: PromptBlockDataNullishType; options: PromptOptionsNullishType; @@ -3988,10 +4231,10 @@ export type InvokeParentType = object_id: string; row_ids?: | /** - * Identifiers for the row to to log a subspan under + * Identifiers for the row to log a subspan under */ /** - * Identifiers for the row to to log a subspan under + * Identifiers for the row to log a subspan under */ (| { /** @@ -4008,7 +4251,7 @@ export type InvokeParentType = root_span_id: string; } /** - * Identifiers for the row to to log a subspan under + * Identifiers for the row to log a subspan under */ | null ) @@ -4056,6 +4299,10 @@ export type InvokeFunctionType = */ FunctionIdType & Partial<{ + /** + * A label identifying the caller, used to attribute usage + */ + request_source: string; /** * Argument to the function, which can be any JSON serializable value */ @@ -4065,14 +4312,14 @@ export type InvokeFunctionType = */ expected: unknown; /** - * Any relevant metadata. This will be logged and available as the `metadata` argument. + * Any relevant metadata to log on the span. For prompts, this is available as the `metadata` template variable. For code functions, this is passed through hooks and is not added to `input`. */ metadata: /** - * Any relevant metadata. This will be logged and available as the `metadata` argument. + * Any relevant metadata to log on the span. For prompts, this is available as the `metadata` template variable. For code functions, this is passed through hooks and is not added to `input`. */ | {} /** - * Any relevant metadata. This will be logged and available as the `metadata` argument. + * Any relevant metadata to log on the span. For prompts, this is available as the `metadata` template variable. For code functions, this is passed through hooks and is not added to `input`. */ | null; /** @@ -4323,6 +4570,14 @@ export type TraceScopeType = { }; export type OnlineScoreConfigType = { status?: AutomationStatusType | undefined; + service_tier?: + | /** + * Service tier for online scoring. If omitted, prompt-level service tiers are preserved. + * + * @enum standard, flex + */ + ("standard" | "flex") + | undefined; /** * The sampling rate for online scoring * @@ -4406,11 +4661,76 @@ export type OrganizationType = { * Name of the organization */ name: string; - api_url?: (string | null) | undefined; - is_universal_api?: (boolean | null) | undefined; - is_dataplane_private?: (boolean | null) | undefined; - proxy_url?: (string | null) | undefined; - realtime_url?: (string | null) | undefined; + api_url?: + | /** + * URL of the organization's data plane API. If unset, the Braintrust-hosted API is used. + */ + /** + * URL of the organization's data plane API. If unset, the Braintrust-hosted API is used. + */ + (| string + /** + * URL of the organization's data plane API. If unset, the Braintrust-hosted API is used. + */ + | null + ) + | undefined; + is_universal_api?: + | /** + * Whether the data plane at `api_url` also serves the AI proxy, at `api_url` with `/v1/proxy` appended + */ + /** + * Whether the data plane at `api_url` also serves the AI proxy, at `api_url` with `/v1/proxy` appended + */ + (| boolean + /** + * Whether the data plane at `api_url` also serves the AI proxy, at `api_url` with `/v1/proxy` appended + */ + | null + ) + | undefined; + is_dataplane_private?: + | /** + * Whether the organization's data plane is only reachable from a private network + */ + /** + * Whether the organization's data plane is only reachable from a private network + */ + (| boolean + /** + * Whether the organization's data plane is only reachable from a private network + */ + | null + ) + | undefined; + proxy_url?: + | /** + * URL of the organization's AI proxy + */ + /** + * URL of the organization's AI proxy + */ + (| string + /** + * URL of the organization's AI proxy + */ + | null + ) + | undefined; + realtime_url?: + | /** + * URL of the organization's realtime service + */ + /** + * URL of the organization's realtime service + */ + (| string + /** + * URL of the organization's realtime service + */ + | null + ) + | undefined; created?: | /** * Date of organization creation @@ -4425,8 +4745,34 @@ export type OrganizationType = { | null ) | undefined; - archived_at?: (string | null) | undefined; - deleted_at?: (string | null) | undefined; + archived_at?: + | /** + * Date the organization was archived + */ + /** + * Date the organization was archived + */ + (| string + /** + * Date the organization was archived + */ + | null + ) + | undefined; + deleted_at?: + | /** + * Date the organization was deleted + */ + /** + * Date the organization was deleted + */ + (| string + /** + * Date the organization was deleted + */ + | null + ) + | undefined; image_rendering_mode?: ImageRenderingModeType | undefined; }; export type RetentionObjectTypeType = @@ -4631,6 +4977,100 @@ export type ProjectSettingsType = Partial<{ * If true, automatically complete a review after every assigned reviewer fills all human review scores visible to them. */ | null; + user_id_fields: Array< + /** + * Path to the application's end-user identifier in metadata or span_attributes, for example ["metadata", "customer_id"]. This identifies an end user, not a session, request, organization, or the Braintrust user who created the project. + */ + Array + > | null; + /** + * Project-wide Slack, webhook, and PagerDuty destinations for new and reactivated Pattern alerts. An empty array disables alerts. + */ + pattern_alert_destinations: /** + * Project-wide Slack, webhook, and PagerDuty destinations for new and reactivated Pattern alerts. An empty array disables alerts. + */ + | Array< + /** + * A delivery action available to an automation + */ + | { + /** + * The type of action to take + * + * @enum webhook + */ + type: "webhook"; + /** + * The webhook URL to send the request to + */ + url: string; + formatting_prompt?: /** + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ + string | undefined; + } + | { + /** + * The type of action to take + * + * @enum slack + */ + type: "slack"; + /** + * The Slack workspace ID to post to + */ + workspace_id: string; + /** + * The Slack channel ID to post to + */ + channel: string; + message_template?: /** + * Custom message template for the alert + */ + string | undefined; + formatting_prompt?: /** + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ + string | undefined; + } + | { + /** + * The type of action to take + * + * @enum pagerduty + */ + type: "pagerduty"; + /** + * The data-plane secret containing the PagerDuty routing key + */ + routing_key_secret_name: string; + /** + * @enum critical, error, warning, info + */ + severity: "critical" | "error" | "warning" | "info"; + formatting_prompt?: /** + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ + string | undefined; + } + > + /** + * Project-wide Slack, webhook, and PagerDuty destinations for new and reactivated Pattern alerts. An empty array disables alerts. + */ + | null; default_preprocessor: NullableSavedFunctionIdType; }> | null; export type ProjectType = { @@ -4715,23 +5155,27 @@ export type WindowedAutomationConfigType = { | /** * The product surface that created and manages the automation * - * @enum patterns + * @enum patterns, dashboard, digest */ (| /** * The product surface that created and manages the automation * - * @enum patterns + * @enum patterns, dashboard, digest */ - "patterns" + ("patterns" | "dashboard" | "digest") /** * The product surface that created and manages the automation * - * @enum patterns + * @enum patterns, dashboard, digest */ | null ) | undefined; status?: AutomationStatusType | undefined; + continuous?: /** + * Between scheduled runs, check for accumulated new work. The executor decides how to process it. + */ + boolean | undefined; threshold?: /** * Optional calculation and lifecycle policy that gate scheduled delivery */ @@ -4899,6 +5343,14 @@ export type WindowedAutomationConfigType = { * @maxLength 10000 */ prompt: string; + digest?: + | { + /** + * @enum compact, detailed + */ + writing_style: "compact" | "detailed"; + } + | undefined; /** * Whether to include the automation trigger payload as input * @@ -4922,14 +5374,14 @@ export type WindowedAutomationConfigType = { string>; harness?: | /** - * Required when saving a Loop automation. Currently only Codex is supported. + * Required for an explicit model selection. Currently only Codex is supported. * - * @enum codex, native, claude-code + * @enum native, codex, claude-code, pi */ - ("codex" | "native" | "claude-code") + ("native" | "codex" | "claude-code" | "pi") | undefined; model?: /** - * Required when saving a Loop automation. + * Omit harness, model, and endpoint_name to inherit the organization default provider and its recommended execution settings. * * @minLength 1 */ @@ -4944,6 +5396,20 @@ export type WindowedAutomationConfigType = { */ ("none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max") | undefined; + target?: /** + * The object this automation writes to or maintains + */ + | { + /** + * @enum dashboard + */ + type: "dashboard"; + /** + * @enum overview + */ + id: "overview"; + } + | undefined; } | undefined; /** @@ -4994,22 +5460,11 @@ export type WindowedAutomationConfigType = { */ string | undefined; formatting_prompt?: /** - * Publish a Slack mrkdwn digest. - - Pattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say "No current pattern outcomes." This requirement overrides conflicting formatting guidance. - - Include a complete "*Pattern outcomes*" section with one row for every included Pattern. - - Use this row format exactly: - • — `outcome` - - If a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern. - - After the outcome list, include a "*Highlights*" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report. - * - * @minLength 1 - * @maxLength 10000 - */ + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ string | undefined; } | { @@ -5243,22 +5698,11 @@ export type TopicDigestAutomationConfigType = { */ string | undefined; formatting_prompt?: /** - * Publish a Slack mrkdwn digest. - - Pattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say "No current pattern outcomes." This requirement overrides conflicting formatting guidance. - - Include a complete "*Pattern outcomes*" section with one row for every included Pattern. - - Use this row format exactly: - • — `outcome` - - If a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern. - - After the outcome list, include a "*Highlights*" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report. - * - * @minLength 1 - * @maxLength 10000 - */ + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ string | undefined; }; topic_map_function_ids?: /** @@ -5387,22 +5831,11 @@ export type ProjectAutomationType = { */ string | undefined; formatting_prompt?: /** - * Publish a Slack mrkdwn digest. - - Pattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say "No current pattern outcomes." This requirement overrides conflicting formatting guidance. - - Include a complete "*Pattern outcomes*" section with one row for every included Pattern. - - Use this row format exactly: - • — `outcome` - - If a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern. - - After the outcome list, include a "*Highlights*" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report. - * - * @minLength 1 - * @maxLength 10000 - */ + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ string | undefined; } | { @@ -5659,22 +6092,11 @@ export type ProjectAutomationType = { */ string | undefined; formatting_prompt?: /** - * Publish a Slack mrkdwn digest. - - Pattern selection requirement: include every selected Pattern whose final status is active at the end of this automation run, plus every selected Pattern that changed from active to inactive during this run (a newly inactive outcome). Exclude Patterns that were already inactive before this run and remained inactive. Do not mention excluded Patterns anywhere in the digest, including Highlights. If no selected Patterns meet these criteria, say "No current pattern outcomes." This requirement overrides conflicting formatting guidance. - - Include a complete "*Pattern outcomes*" section with one row for every included Pattern. - - Use this row format exactly: - • — `outcome` - - If a Pattern has no URL, use the plain title instead. Do not use GitHub Markdown tables or code-block tables, because links must remain clickable. Do not omit any active selected Pattern. - - After the outcome list, include a "*Highlights*" section with one very short paragraph, 2-3 sentences maximum. Summarize what changed or what broadly stands out among the included active Patterns. Do not introduce new claims beyond the run report. - * - * @minLength 1 - * @maxLength 10000 - */ + * Instructions for Loop to format content sent to this destination + * + * @minLength 1 + * @maxLength 10000 + */ string | undefined; } | { @@ -5783,7 +6205,7 @@ export type ProjectGroupType = { ) | undefined; /** - * Sorted ids of active projects in this project group + * Sorted ids of active projects in this project group. Modifying membership requires project_group:update on the group. */ member_projects: Array; }; @@ -5846,14 +6268,14 @@ export type ProjectLogsEventType = { unknown | undefined; scores?: | /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. */ /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. */ (| {} /** - * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. + * A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and ground truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. */ | null ) @@ -5971,7 +6393,7 @@ export type ProjectLogsEventType = { ) | undefined; /** - * A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing + * A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://docs.braintrust.dev/instrument) for full details on tracing */ span_id: string; span_parents?: @@ -6620,7 +7042,20 @@ export type RunEvalType = { | null ) | undefined; - _internal_btql?: ({} | null) | undefined; + _internal_btql?: + | /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + (| {} + /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + | null + ) + | undefined; } /** * Project and dataset name @@ -6656,13 +7091,30 @@ export type RunEvalType = { | null ) | undefined; - _internal_btql?: ({} | null) | undefined; + _internal_btql?: + | /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + (| {} + /** + * Internal BTQL query overrides in parsed JSON form that select which dataset rows to evaluate. `filter` is a BTQL expression object, `sort` is an array of BTQL sort expressions, and `limit` is an integer. + */ + | null + ) + | undefined; } /** * Dataset rows */ | { data: Array; + source_dataset_id?: /** + * The saved dataset these rows came from. Its project scopes saved simulated-user references; the caller must have read access to the dataset and source project. + */ + string | undefined; } /** * Experiment whose inputs and outputs should be used as dataset inputs and expected values @@ -6734,14 +7186,14 @@ export type RunEvalType = { | undefined; timeout?: | /** - * The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout. + * The maximum duration, in milliseconds, to run the evaluation. There is no timeout by default. */ /** - * The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout. + * The maximum duration, in milliseconds, to run the evaluation. There is no timeout by default. */ (| number /** - * The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout. + * The maximum duration, in milliseconds, to run the evaluation. There is no timeout by default. */ | null ) @@ -6914,6 +7366,9 @@ export type ServiceTokenType = { * Name of the service token */ name: string; + /** + * Masked preview showing the token prefix and last four characters, e.g. bt-st-…AbCd. + */ preview_name: string; service_account_id?: | /** @@ -7229,6 +7684,10 @@ export type ViewOptionsType = */ viewType: "monitor"; options: Partial<{ + /** + * @enum span, trace + */ + filterScope: "span" | "trace"; /** * @enum range, frame */ @@ -7359,7 +7818,7 @@ export type ViewType = { /** * Type of object that the view corresponds to. * - * @enum projects, experiments, experiment, playgrounds, playground, datasets, dataset, prompts, parameters, tools, scorers, classifiers, logs, monitor, for_review_project_log, for_review_experiments, for_review_datasets + * @enum projects, experiments, experiment, playgrounds, playground, datasets, dataset, prompts, parameters, tools, scorers, classifiers, logs, patterns, monitor, for_review_project_log, for_review_experiments, for_review_datasets */ view_type: | "projects" @@ -7375,6 +7834,7 @@ export type ViewType = { | "scorers" | "classifiers" | "logs" + | "patterns" | "monitor" | "for_review_project_log" | "for_review_experiments" diff --git a/js/util/generated_types.ts b/js/util/generated_types.ts index ea2bc045b..7c1aff8e5 100644 --- a/js/util/generated_types.ts +++ b/js/util/generated_types.ts @@ -1,4 +1,4 @@ -// Auto-generated file (content hash 387034529ea006a9) -- do not modify +// Auto-generated file (content hash 986b8a9b0d902f2d) -- do not modify import { z } from "zod/v3"; @@ -191,7 +191,11 @@ export const AsyncScoringState = z.union([ ]); export type AsyncScoringStateType = z.infer; export const AsyncScoringControl = z.union([ - z.object({ kind: z.literal("score_update"), token: z.string().optional() }), + z.object({ + kind: z.literal("score_update"), + token: z.string().optional(), + triggered_xact_id: z.string().optional(), + }), z.object({ kind: z.literal("state_override"), state: AsyncScoringState }), z.object({ kind: z.literal("state_force_reselect") }), z.object({ kind: z.literal("state_enabled_force_rescore") }), @@ -829,6 +833,7 @@ export const SpanType = z.union([ "preprocessor", "classifier", "review", + "decision", "question", "log", ]), @@ -970,6 +975,46 @@ export const PromptBlockDataNullish = z.union([ tools: z.string().optional(), }), z.object({ type: z.literal("completion"), content: z.string() }), + z.object({ + type: z.literal("decisions"), + input: z.union([ + z.string(), + z.array( + z.object({ + role: z.enum(["system", "developer", "user", "assistant"]), + content: z.union([ + z.string(), + z.array(ChatCompletionContentPartText), + ]), + }), + ), + ]), + questions: z.array( + z.object({ name: z.string().optional(), instructions: z.string() }).and( + z.union([ + z.object({ type: z.literal("predicate") }), + z.object({ + type: z.literal("choice"), + choices: z.array( + z.object({ + value: z.union([z.string(), z.boolean()]), + description: z.string().optional(), + }), + ), + }), + z.object({ + type: z.literal("score"), + levels: z.array( + z.object({ + label: z.string(), + description: z.string().optional(), + }), + ), + }), + ]), + ), + ), + }), z.null(), ]); export type PromptBlockDataNullishType = z.infer; @@ -1173,6 +1218,46 @@ export const PromptBlockData = z.union([ tools: z.string().optional(), }), z.object({ type: z.literal("completion"), content: z.string() }), + z.object({ + type: z.literal("decisions"), + input: z.union([ + z.string(), + z.array( + z.object({ + role: z.enum(["system", "developer", "user", "assistant"]), + content: z.union([ + z.string(), + z.array(ChatCompletionContentPartText), + ]), + }), + ), + ]), + questions: z.array( + z.object({ name: z.string().optional(), instructions: z.string() }).and( + z.union([ + z.object({ type: z.literal("predicate") }), + z.object({ + type: z.literal("choice"), + choices: z.array( + z.object({ + value: z.union([z.string(), z.boolean()]), + description: z.string().optional(), + }), + ), + }), + z.object({ + type: z.literal("score"), + levels: z.array( + z.object({ + label: z.string(), + description: z.string().optional(), + }), + ), + }), + ]), + ), + ), + }), ]); export type PromptBlockDataType = z.infer; export const GraphNode = z.union([ @@ -1297,6 +1382,68 @@ export const FunctionData = z.union([ }), }), TopicMapData.and(z.unknown()), + z.object({ + type: z.literal("simulation"), + config: z.union([ + z.object({ + initiator: z + .enum(["agent", "simulated_user"]) + .optional() + .default("agent"), + timeout_seconds: z + .number() + .int() + .gte(15) + .lte(3600) + .optional() + .default(600), + modality: z.literal("voice").optional().default("voice"), + agent: z.union([ + z.object({ + type: z.literal("livekit"), + server_url: z.string().min(1), + agent_name: z.string().min(1), + }), + z.object({ + type: z.literal("phone"), + phone_number: z.string().regex(/^\+[1-9]\d{7,14}$/), + }), + z.object({ + type: z.literal("sip"), + uri: z.string().regex(/^sips?:[^\s]+$/), + }), + ]), + }), + z.object({ + initiator: z + .enum(["agent", "simulated_user"]) + .optional() + .default("agent"), + timeout_seconds: z + .number() + .int() + .gte(15) + .lte(3600) + .optional() + .default(600), + modality: z.literal("text"), + agent: z.union([ + z.object({ + type: z.literal("livekit"), + server_url: z.string().min(1), + agent_name: z.string().min(1), + }), + z.object({ + type: z.literal("acp"), + command: z.string().min(1), + args: z.array(z.string()).optional().default([]), + cwd: z.string().regex(/^(?:[\/]|[A-Za-z]:[\\\/]|\\\\)/), + permissions: z.literal("deny").optional().default("deny"), + }), + ]), + }), + ]), + }), ]); export type FunctionDataType = z.infer; export const Function = z.object({ @@ -1340,6 +1487,7 @@ export const FunctionFormat = z.enum([ "global", "graph", "topic_map", + "simulation", ]); export type FunctionFormatType = z.infer; export const PromptData = z @@ -1521,6 +1669,7 @@ export type StreamingModeType = z.infer; export const InvokeFunction = FunctionId.and( z .object({ + request_source: z.string(), input: z.unknown(), expected: z.unknown(), metadata: z.union([z.object({}).partial().passthrough(), z.null()]), @@ -1605,6 +1754,7 @@ export type TraceScopeType = z.infer; export const OnlineScoreConfig = z.union([ z.object({ status: AutomationStatus.optional(), + service_tier: z.enum(["standard", "flex"]).optional(), sampling_rate: z.number().gte(0).lte(1), scorers: z.array(SavedFunctionId), btql_filter: z.union([z.string(), z.null()]).optional(), @@ -1683,6 +1833,35 @@ export const ProjectSettings = z.union([ coding_agent_insights_dashboard: z.union([z.boolean(), z.null()]), blind_reviews: z.union([z.boolean(), z.null()]), require_all_human_review_scores: z.union([z.boolean(), z.null()]), + user_id_fields: z.union([ + z.array(z.array(z.string().min(1).max(256)).min(2).max(32)), + z.null(), + ]), + pattern_alert_destinations: z.union([ + z.array( + z.union([ + z.object({ + type: z.literal("webhook"), + url: z.string(), + formatting_prompt: z.string().min(1).max(10000).optional(), + }), + z.object({ + type: z.literal("slack"), + workspace_id: z.string(), + channel: z.string(), + message_template: z.string().optional(), + formatting_prompt: z.string().min(1).max(10000).optional(), + }), + z.object({ + type: z.literal("pagerduty"), + routing_key_secret_name: z.string().uuid(), + severity: z.enum(["critical", "error", "warning", "info"]), + formatting_prompt: z.string().min(1).max(10000).optional(), + }), + ]), + ), + z.null(), + ]), default_preprocessor: NullableSavedFunctionId, }) .partial(), @@ -1702,8 +1881,11 @@ export const Project = z.object({ export type ProjectType = z.infer; export const WindowedAutomationConfig = z.object({ event_type: z.literal("windowed"), - product_origin: z.union([z.literal("patterns"), z.null()]).optional(), + product_origin: z + .union([z.enum(["patterns", "dashboard", "digest"]), z.null()]) + .optional(), status: AutomationStatus.optional(), + continuous: z.boolean().optional(), threshold: z .object({ calculation: z.object({ @@ -1745,15 +1927,21 @@ export const WindowedAutomationConfig = z.object({ loop: z .object({ prompt: z.string().min(1).max(10000), + digest: z + .object({ writing_style: z.enum(["compact", "detailed"]) }) + .optional(), include_trigger_input: z.boolean().optional().default(false), agent_slug: z.string().min(1), auto_approve_tools: z.array(z.string().min(1)).optional().default([]), - harness: z.enum(["codex", "native", "claude-code"]).optional(), + harness: z.enum(["native", "codex", "claude-code", "pi"]).optional(), model: z.string().min(1).optional(), endpoint_name: z.string().min(1).optional(), reasoning_effort: z .enum(["none", "minimal", "low", "medium", "high", "xhigh", "max"]) .optional(), + target: z + .object({ type: z.literal("dashboard"), id: z.literal("overview") }) + .optional(), }) .optional(), actions: z @@ -2240,7 +2428,10 @@ export const RunEval = z.object({ .union([z.object({}).partial().passthrough(), z.null()]) .optional(), }), - z.object({ data: z.array(z.unknown()) }), + z.object({ + data: z.array(z.unknown()), + source_dataset_id: z.string().optional(), + }), z.object({ experiment_name: z.string() }), ]), name: z.string().optional(), @@ -2375,6 +2566,7 @@ export const ViewOptions = z.union([ viewType: z.literal("monitor"), options: z .object({ + filterScope: z.enum(["span", "trace"]), spanType: z.union([z.enum(["range", "frame"]), z.null()]), rangeValue: z.union([z.string(), z.null()]), frameStart: z.union([z.string(), z.null()]), @@ -2475,6 +2667,7 @@ export const View = z.object({ "scorers", "classifiers", "logs", + "patterns", "monitor", "for_review_project_log", "for_review_experiments",