diff --git a/.changeset/span-cache-flush-errors.md b/.changeset/span-cache-flush-errors.md new file mode 100644 index 000000000..2a6de5282 --- /dev/null +++ b/.changeset/span-cache-flush-errors.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix: Don't crash the process when the span cache can't write its temp file during an eval diff --git a/js/src/span-cache-flush-failure.test.ts b/js/src/span-cache-flush-failure.test.ts new file mode 100644 index 000000000..fb49ca850 --- /dev/null +++ b/js/src/span-cache-flush-failure.test.ts @@ -0,0 +1,59 @@ +import { afterEach, beforeAll, describe, expect, test } from "vite-plus/test"; +import { SpanCache } from "./span-cache"; +import { configureNode } from "./node/config"; + +// Eval() turns the span cache on and every span write goes through +// queueWrite(). If the temp file cannot be created (read-only root +// filesystem, deleted tmpdir) the background flush rejects. Nothing awaits +// that promise, so Node ends the process with an unhandled rejection. +describe("SpanCache background flush failures", () => { + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => { + unhandled.push(reason); + }; + const realTmpdir = process.env.TMPDIR; + let cache: SpanCache | undefined; + + beforeAll(() => { + configureNode(); + }); + + afterEach(() => { + process.off("unhandledRejection", onUnhandled); + if (realTmpdir === undefined) delete process.env.TMPDIR; + else process.env.TMPDIR = realTmpdir; + cache?.stop(); + cache?.dispose(); + cache = undefined; + unhandled.length = 0; + }); + + test("an unwritable temp dir does not cause an unhandled rejection", async () => { + process.on("unhandledRejection", onUnhandled); + process.env.TMPDIR = "/nonexistent-braintrust-tmp-dir"; + + cache = new SpanCache(); + cache.start(); + cache.queueWrite("root-1", "span-1", { span_id: "span-1", input: "x" }); + await new Promise((resolve) => setTimeout(resolve, 100)); + cache.queueWrite("root-2", "span-2", { span_id: "span-2", input: "y" }); + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(unhandled).toEqual([]); + }); + + test("spans queued while the temp dir is unwritable stay readable from memory", async () => { + process.on("unhandledRejection", onUnhandled); + process.env.TMPDIR = "/nonexistent-braintrust-tmp-dir"; + + cache = new SpanCache(); + cache.start(); + cache.queueWrite("root-1", "span-1", { span_id: "span-1", input: "x" }); + await new Promise((resolve) => setTimeout(resolve, 100)); + + expect(cache.getByRootSpanId("root-1")).toEqual([ + { span_id: "span-1", input: "x" }, + ]); + expect(unhandled).toEqual([]); + }); +}); diff --git a/js/src/span-cache.ts b/js/src/span-cache.ts index 7b197d40e..22eb5cfe8 100644 --- a/js/src/span-cache.ts +++ b/js/src/span-cache.ts @@ -10,6 +10,7 @@ */ import iso from "#braintrust-platform"; +import { debugLogger } from "./debug-logger"; import { mergeDicts } from "../util/object_util"; // Global registry of active span caches for process exit cleanup @@ -275,14 +276,26 @@ export class SpanCache { return; } - await this.ensureInitialized(); + try { + await this.ensureInitialized(); + } catch (err) { + // Don't cache a rejected initialization forever: allow a later retry. + this.initPromise = null; + debugLogger.warn("Failed to initialize span cache file:", err); + return; + } if (generation !== this.generation || !this.fileHandle) { return; } const lines = recordsToFlush.map((r) => JSON.stringify(r) + "\n").join(""); - await this.fileHandle.appendFile(lines, "utf8"); + try { + await this.fileHandle.appendFile(lines, "utf8"); + } catch (err) { + debugLogger.warn("Failed to flush span cache writes:", err); + return; + } // Only now remove the flushed records from the buffer. // Filter out the records we just wrote (compare by reference).