feat: Add utilities for custom views (bt custom-views push & bt custom-views preview) (#2191)

Author: lforst

.changeset/custom-view-helpers.md

@@ -0,0 +1,5 @@
+---
+"braintrust": minor
+---
+
+feat: Add utilities for custom views (push + preview)

js/package.json

@@ -37,6 +37,13 @@
       "require": "./dist/index.js",
       "default": "./dist/index.mjs"
     },
+    "./custom-views": {
+      "types": "./dist/custom-views.d.ts",
+      "import": "./dist/custom-views.mjs",
+      "module": "./dist/custom-views.mjs",
+      "require": "./dist/custom-views.js",
+      "default": "./dist/custom-views.mjs"
+    },
     "./workerd": {
       "import": "./dist/workerd.mjs",
       "require": "./dist/workerd.js",
@@ -214,6 +221,7 @@
   },
   "dependencies": {
     "@next/env": "^14.2.3",
+    "@types/react": "^18.3.1",
     "@vercel/functions": "^1.0.2",
     "acorn": "^8.16.0",
     "acorn-import-attributes": "^1.9.5",

js/src/custom-views/exports.ts

@@ -0,0 +1 @@
+export { customDatasetView, customTraceView } from "./registry";

js/src/custom-views/registry.ts

@@ -0,0 +1,177 @@
+import type { ReactNode } from "react";
+
+type SpanFieldName = "input" | "output" | "expected" | "metadata";
+
+type SpanFields<
+  TInput = unknown,
+  TOutput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  input?: TInput;
+  output?: TOutput;
+  expected?: TExpected;
+  metadata?: TMetadata;
+};
+
+type TraceViewSpan<
+  TInput = unknown,
+  TOutput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  id: string;
+  span_id: string;
+  root_span_id: string;
+  parent_span_id?: string | null;
+  span_parents?: string[];
+  data: {
+    input?: TInput;
+    output?: TOutput;
+    expected?: TExpected;
+    metadata?: TMetadata;
+    scores?: Record<string, number | null>;
+    metrics?: Record<string, number | string>;
+    error?: string;
+    tags?: string[];
+    span_attributes?: Record<string, unknown>;
+    [key: string]: unknown;
+  };
+  children: string[];
+};
+
+type CustomViewUpdateTarget = "selected" | "root" | { spanId: string };
+
+type CustomViewUpdateResult = {
+  transactionId: string | null;
+};
+
+type CustomViewSpanUpdate<
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  target?: CustomViewUpdateTarget;
+  metadata?: Partial<TMetadata> & Record<string, unknown>;
+  scores?: Record<string, number | null>;
+  tags?: string[] | null;
+};
+
+type TraceViewTrace<
+  TInput = unknown,
+  TOutput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  rootSpanId: string;
+  selectedSpanId: string;
+  spanOrder: string[];
+  spans: Record<string, TraceViewSpan<TInput, TOutput, TExpected, TMetadata>>;
+  fetchSpanFields: (
+    spanIds: string | string[],
+    fields?: SpanFieldName[],
+  ) => Promise<
+    Record<string, SpanFields<TInput, TOutput, TExpected, TMetadata>>
+  >;
+  update: (
+    update: CustomViewSpanUpdate<TMetadata>,
+  ) => Promise<CustomViewUpdateResult>;
+};
+
+type TraceViewProps<
+  TInput = unknown,
+  TOutput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  trace: TraceViewTrace<TInput, TOutput, TExpected, TMetadata>;
+  span: TraceViewSpan<TInput, TOutput, TExpected, TMetadata>;
+  selectSpan?: (spanId: string) => void;
+  update?: (field: string, value: unknown) => void;
+};
+
+type DatasetViewProps<
+  TInput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  id: string;
+  input?: TInput;
+  expected?: TExpected;
+  metadata?: TMetadata;
+  tags?: string[];
+  update: (update: {
+    input?: TInput;
+    expected?: TExpected;
+    metadata?: Partial<TMetadata> & Record<string, unknown>;
+    tags?: string[] | null;
+  }) => Promise<CustomViewUpdateResult>;
+};
+
+type Component<Props> = (props: Props) => ReactNode;
+
+type ProjectRef = string | { id: string } | { name: string };
+type DatasetRef = { id: string } | { name: string };
+
+type CustomTraceViewDefinition = {
+  name: string;
+  slug: string;
+  project?: ProjectRef;
+};
+
+type CustomTraceView<
+  TInput = unknown,
+  TOutput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+> = CustomTraceViewDefinition & {
+  kind: "trace";
+  component: Component<TraceViewProps<TInput, TOutput, TExpected, TMetadata>>;
+};
+
+type CustomDatasetViewDefinition = {
+  name: string;
+  slug: string;
+  dataset: DatasetRef;
+  project?: ProjectRef;
+};
+
+type CustomDatasetView<
+  TInput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+> = CustomDatasetViewDefinition & {
+  kind: "dataset";
+  component: Component<DatasetViewProps<TInput, TExpected, TMetadata>>;
+};
+
+/**
+ * Defines a trace custom view for discovery by the `bt views` CLI.
+ *
+ * @experimental This API is not yet stabilized and may change across non-major versions.
+ */
+export function customTraceView<
+  TInput = unknown,
+  TOutput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+>(
+  definition: CustomTraceViewDefinition,
+  component: Component<TraceViewProps<TInput, TOutput, TExpected, TMetadata>>,
+): CustomTraceView<TInput, TOutput, TExpected, TMetadata> {
+  return { ...definition, component, kind: "trace" };
+}
+
+/**
+ * Defines a dataset custom view for discovery by the `bt views` CLI.
+ *
+ * @experimental This API is not yet stabilized and may change across non-major versions.
+ */
+export function customDatasetView<
+  TInput = unknown,
+  TExpected = unknown,
+  TMetadata extends Record<string, unknown> = Record<string, unknown>,
+>(
+  definition: CustomDatasetViewDefinition,
+  component: Component<DatasetViewProps<TInput, TExpected, TMetadata>>,
+): CustomDatasetView<TInput, TExpected, TMetadata> {
+  return { ...definition, component, kind: "dataset" };
+}

js/tsup.config.ts

@@ -14,6 +14,7 @@ export default defineConfig([
   {
     entry: {
       index: "src/node/index.ts",
+      "custom-views": "src/custom-views/exports.ts",
       "apply-auto-instrumentation":
         "src/node/apply-auto-instrumentation-entry.ts",
       "vitest-evals-reporter": "src/wrappers/vitest-evals/reporter.ts",