Repository navigation
Expand file tree
/
Copy pathspan-cache.ts
More file actions
464 lines (408 loc) · 13 KB
/
Copy pathspan-cache.ts
File metadata and controls
464 lines (408 loc) · 13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
/**
* SpanCache provides a disk-based cache for span data, allowing
* scorers to read spans without making server round-trips when possible.
*
* Spans are stored on disk to minimize memory usage during evaluations.
* The cache file is automatically cleaned up when dispose() is called.
*
* In browser environments where filesystem access isn't available,
* the cache becomes a no-op (all lookups return undefined).
*/
import iso from "#braintrust-platform";
import { debugLogger } from "./debug-logger";
import { mergeDicts } from "../util/object_util";
// Global registry of active span caches for process exit cleanup
const activeCaches = new Set<SpanCache>();
let exitHandlersRegistered = false;
/**
* Check if the span cache can be used (requires filesystem APIs).
* This is called at runtime, not at module load time, to allow
* configureNode() to set up the isomorph functions first.
*/
function canUseSpanCache(): boolean {
return !!(
iso.pathJoin &&
iso.tmpdir &&
iso.writeFileSync &&
iso.appendFileSync &&
iso.readFileSync &&
iso.unlinkSync &&
iso.openFile
);
}
export interface CachedSpan {
input?: unknown;
output?: unknown;
expected?: unknown;
error?: unknown;
scores?: Record<string, unknown>;
metrics?: Record<string, unknown>;
metadata?: Record<string, unknown>;
tags?: string[];
span_id: string;
span_parents?: string[];
is_root?: boolean | null;
span_attributes?: {
name?: string;
type?: string;
[key: string]: unknown;
};
}
interface DiskSpanRecord {
rootSpanId: string;
spanId: string;
data: CachedSpan;
}
/**
* Disk-based cache for span data, keyed by rootSpanId.
*
* This cache writes spans to a temporary file to minimize memory usage.
* It uses append-only writes and reads the full file when querying.
*
* In browser environments, this cache is automatically disabled and
* all operations become no-ops.
*/
export class SpanCache {
private cacheFilePath: string | null = null;
private fileHandle: any | null = null; // type-erased fs.promises.FileHandle
private initialized = false;
private initPromise: Promise<void> | null = null;
private generation = 0;
// Tracks whether the cache was explicitly disabled (via constructor or disable())
private _explicitlyDisabled: boolean;
// Tracks whether the cache has been enabled (for evals only)
private _enabled: boolean = false;
// Reference count of active evals using this cache
private _activeEvalCount: number = 0;
// Small in-memory index tracking which rootSpanIds have data
private rootSpanIndex: Set<string> = new Set();
constructor(options?: { disabled?: boolean }) {
// Track if user explicitly disabled the cache
this._explicitlyDisabled = options?.disabled ?? false;
// Cache is disabled by default until enable() is called (e.g., during Eval)
// Initialization is lazy - file is created on first write
}
/**
* Disable the cache at runtime. This is called automatically when
* initFunction is used, since remote function spans won't be in the cache.
*/
disable(): void {
this._explicitlyDisabled = true;
}
/**
* Start caching spans for use during evaluations.
* This only starts caching if the cache wasn't permanently disabled.
* Called by Eval() to turn on caching for the duration of the eval.
* Uses reference counting to support parallel evals.
*/
start(): void {
if (!this._explicitlyDisabled) {
this._enabled = true;
this._activeEvalCount++;
}
}
/**
* Stop caching spans and return to the default disabled state.
* Unlike disable(), this allows start() to work again for future evals.
* Called after an eval completes to return to the default state.
* Uses reference counting - only disables when all evals are complete.
*/
stop(): void {
this._activeEvalCount--;
if (this._activeEvalCount <= 0) {
this._activeEvalCount = 0;
this._enabled = false;
}
}
get disabled(): boolean {
// Disabled if: explicitly disabled, not enabled, or platform doesn't support it
return this._explicitlyDisabled || !this._enabled || !canUseSpanCache();
}
private async ensureInitialized(): Promise<void> {
if (this.disabled) {
return;
}
if (this.initialized) {
return;
}
if (this.initPromise) {
return this.initPromise;
}
const generation = this.generation;
this.initPromise = (async () => {
if (!iso.tmpdir || !iso.pathJoin || !iso.openFile) {
// Filesystem not available - silently skip initialization
return;
}
const tmpDir = iso.tmpdir();
const uniqueId = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
const cacheFilePath = iso.pathJoin(
tmpDir,
`braintrust-span-cache-${uniqueId}.jsonl`,
);
// Open file for append+read
const fileHandle = await iso.openFile(cacheFilePath, "a+");
if (generation !== this.generation) {
try {
await fileHandle.close();
} catch {
// Ignore cleanup errors after disposal.
}
if (iso.unlinkSync) {
try {
iso.unlinkSync(cacheFilePath);
} catch {
// Ignore cleanup errors after disposal.
}
}
return;
}
this.cacheFilePath = cacheFilePath;
this.fileHandle = fileHandle;
this.initialized = true;
// Register cleanup handler on first initialization
this.registerExitHandler();
})();
return this.initPromise;
}
/**
* Register a handler to clean up the temp file on process exit.
* Uses a global registry to avoid registering multiple handlers.
*/
private registerExitHandler(): void {
// Add this cache to the global registry
activeCaches.add(this);
// Only register process handlers once globally
if (
typeof process !== "undefined" &&
process.on &&
!exitHandlersRegistered
) {
exitHandlersRegistered = true;
const cleanupAllCaches = () => {
// Clean up all active caches
for (const cache of activeCaches) {
// Close file handle if open
if (cache.fileHandle) {
try {
cache.fileHandle.close().catch(() => {});
cache.fileHandle = null;
} catch {
// Ignore errors during exit cleanup
}
}
// Delete the temp file
if (cache.cacheFilePath && canUseSpanCache() && iso.unlinkSync) {
try {
iso.unlinkSync(cache.cacheFilePath);
} catch {
// Ignore cleanup errors - file might not exist or already deleted
}
}
}
};
// Register for multiple exit scenarios
process.on("exit", cleanupAllCaches);
process.on("SIGINT", cleanupAllCaches);
process.on("SIGTERM", cleanupAllCaches);
process.on("beforeExit", cleanupAllCaches);
}
}
// Buffer for pending writes - flushed asynchronously
private writeBuffer: DiskSpanRecord[] = [];
private flushScheduled = false;
private flushPromise: Promise<void> | null = null;
/**
* Queue a span write for async flushing.
* This is non-blocking - writes are buffered in memory and flushed
* to disk on the next microtask.
*/
queueWrite(rootSpanId: string, spanId: string, data: CachedSpan): void {
if (this.disabled) {
return;
}
const record: DiskSpanRecord = { rootSpanId, spanId, data };
this.writeBuffer.push(record);
this.rootSpanIndex.add(rootSpanId);
// Schedule async flush if not already scheduled
if (!this.flushScheduled) {
this.flushScheduled = true;
this.flushPromise = this.flushWriteBuffer();
}
}
/**
* Flush the write buffer to disk asynchronously.
* Called automatically after queueWrite, but can also be called explicitly.
*/
async flushWriteBuffer(): Promise<void> {
const generation = this.generation;
// Take a snapshot of records to flush, but DON'T clear the buffer yet.
// Records stay in writeBuffer until disk write succeeds so getByRootSpanId can find them.
const recordsToFlush = [...this.writeBuffer];
this.flushScheduled = false;
if (recordsToFlush.length === 0) {
return;
}
try {
await this.ensureInitialized();
} catch (err) {
// Don't cache a rejected initialization forever: allow a later retry.
this.initPromise = null;
debugLogger.warn("Failed to initialize span cache file:", err);
return;
}
if (generation !== this.generation || !this.fileHandle) {
return;
}
const lines = recordsToFlush.map((r) => JSON.stringify(r) + "\n").join("");
try {
await this.fileHandle.appendFile(lines, "utf8");
} catch (err) {
debugLogger.warn("Failed to flush span cache writes:", err);
return;
}
// Only now remove the flushed records from the buffer.
// Filter out the records we just wrote (compare by reference).
this.writeBuffer = this.writeBuffer.filter(
(r) => !recordsToFlush.includes(r),
);
}
/**
* Wait for any pending writes to complete.
* Call this before reading from the cache to ensure consistency.
*/
async waitForPendingWrites(): Promise<void> {
if (this.flushPromise) {
await this.flushPromise;
this.flushPromise = null;
}
}
/**
* Get all cached spans for a given rootSpanId.
*
* This reads the file and merges all records for the given rootSpanId.
*
* @param rootSpanId The root span ID to look up
* @returns Array of cached spans, or undefined if not in cache
*/
getByRootSpanId(rootSpanId: string): CachedSpan[] | undefined {
if (this.disabled) {
return undefined;
}
// Quick check using in-memory index
if (!this.rootSpanIndex.has(rootSpanId)) {
return undefined;
}
// Accumulate spans by spanId, merging updates
const spanMap = new Map<string, CachedSpan>();
// First, read from disk if initialized
if (this.initialized && this.cacheFilePath && iso.readFileSync) {
try {
const content = iso.readFileSync(this.cacheFilePath, "utf8");
const lines = content.trim().split("\n").filter(Boolean);
for (const line of lines) {
try {
const record: DiskSpanRecord = JSON.parse(line);
if (record.rootSpanId !== rootSpanId) {
continue;
}
const existing = spanMap.get(record.spanId);
if (existing) {
mergeDicts(
existing as unknown as Record<string, unknown>,
record.data as unknown as Record<string, unknown>,
);
} else {
spanMap.set(record.spanId, record.data);
}
} catch {
// Skip malformed lines
}
}
} catch {
// Continue to check buffer even if disk read fails
}
}
// Also check the in-memory write buffer for unflushed data
for (const record of this.writeBuffer) {
if (record.rootSpanId !== rootSpanId) {
continue;
}
const existing = spanMap.get(record.spanId);
if (existing) {
mergeDicts(
existing as unknown as Record<string, unknown>,
record.data as unknown as Record<string, unknown>,
);
} else {
spanMap.set(record.spanId, record.data);
}
}
if (spanMap.size === 0) {
return undefined;
}
return Array.from(spanMap.values());
}
/**
* Check if a rootSpanId has cached data.
*/
has(rootSpanId: string): boolean {
if (this.disabled) {
return false;
}
return this.rootSpanIndex.has(rootSpanId);
}
/**
* Clear all cached spans for a given rootSpanId.
* Note: This only removes from the index. The data remains in the file
* but will be ignored on reads.
*/
clear(rootSpanId: string): void {
this.rootSpanIndex.delete(rootSpanId);
}
/**
* Clear all cached data and remove the cache file.
*/
clearAll(): void {
this.rootSpanIndex.clear();
this.dispose();
}
/**
* Get the number of root spans currently tracked.
*/
get size(): number {
return this.rootSpanIndex.size;
}
/**
* Clean up the cache file. Call this when the eval is complete.
* Only performs cleanup when all active evals have completed (refcount = 0).
*/
dispose(): void {
// Only dispose if no active evals are using this cache
if (this._activeEvalCount > 0) {
return;
}
// Remove from global registry
activeCaches.delete(this);
this.generation++;
// Clear pending writes
this.writeBuffer = [];
this.flushScheduled = false;
this.flushPromise = null;
if (this.fileHandle) {
this.fileHandle.close().catch(() => {});
this.fileHandle = null;
}
if (this.cacheFilePath && canUseSpanCache() && iso.unlinkSync) {
try {
iso.unlinkSync(this.cacheFilePath);
} catch {
// Ignore cleanup errors
}
this.cacheFilePath = null;
}
this.initialized = false;
this.initPromise = null;
this.rootSpanIndex.clear();
}
}