@@ -13,11 +13,78 @@ import {
1313} from "./telemetry-file-utils.js" ;
1414
1515export const LOOP_PERF_RELATIVE_PATH = "perf.jsonl" ;
16+ const LOOP_PERF_COMMAND_MAX_BYTES = 64 ;
17+ const LOOP_PERF_PARSE_FAILURE_RAW_BYTES_MAX_BYTES = 1024 ;
18+ const LOOP_PERF_PARSE_FAILURE_ERROR_MESSAGE_MAX_BYTES = 512 ;
19+ const LOOP_PERF_PARSE_FAILURE_MAX_EVENTS_PER_CHUNK = 20 ;
20+
21+ // biome-ignore lint/complexity/useRegexLiterals: Control characters (\u001b, \u009b) required for ANSI stripping
22+ const ANSI_RE = new RegExp (
23+ String . raw `[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]` ,
24+ "g" ,
25+ ) ;
26+ const CONTROL_CHARS_RE = / [ \u0000 - \u001f \u007f - \u009f ] / g;
27+ const CREDENTIAL_RE =
28+ / (?: [ " ' ] ? \b (?: a u t h o r i z a t i o n | p a s s w o r d | (?: [ a - z 0 - 9 ] + [ _ - ] ) * t o k e n | (?: [ a - z 0 - 9 ] + [ _ - ] ) * a p i [ _ - ] ? k e y | (?: [ a - z 0 - 9 ] + [ _ - ] ) * s e c r e t ) \b [ " ' ] ? \s * (?: : | = | \s + ) \s * [ " ' ] ? \S + | \b b e a r e r \s + \S + | \b s k [ - _ ] [ a - z 0 - 9 ] + | \b g h [ p o u s r ] _ [ a - z 0 - 9 _ ] + | \b x o x [ a b p r s ] - [ a - z 0 - 9 - ] + ) / i;
1629
1730function getLoopPerfTelemetryFilePath ( workdir : string ) : string {
1831 return path . join ( workdir , LOOP_PERF_RELATIVE_PATH ) ;
1932}
2033
34+ function truncateUtf8 ( input : string , maxBytes : number ) : string {
35+ const encoded = new TextEncoder ( ) . encode ( input ) ;
36+ if ( encoded . length <= maxBytes ) {
37+ return input ;
38+ }
39+ let end = maxBytes ;
40+ while ( end > 0 && encoded [ end ] !== undefined && encoded [ end ] >= 0x80 && encoded [ end ] <= 0xbf ) {
41+ end -= 1 ;
42+ }
43+ return new TextDecoder ( ) . decode ( encoded . subarray ( 0 , end ) ) ;
44+ }
45+
46+ function stripUnsafeText ( input : string ) : string {
47+ return input . replaceAll ( ANSI_RE , "" ) . replaceAll ( CONTROL_CHARS_RE , "" ) ;
48+ }
49+
50+ function redactCredentialLikeText ( input : string ) : string {
51+ return CREDENTIAL_RE . test ( input ) ? "[redacted]" : input ;
52+ }
53+
54+ function sanitizeLoopPerfCommand ( command : string | null | undefined ) : string | undefined {
55+ if ( command === null || command === undefined ) {
56+ return undefined ;
57+ }
58+ const stripped = stripUnsafeText ( command ) . trim ( ) ;
59+ if ( stripped . length === 0 ) {
60+ return undefined ;
61+ }
62+ return truncateUtf8 (
63+ redactCredentialLikeText ( stripped ) ,
64+ LOOP_PERF_COMMAND_MAX_BYTES ,
65+ ) ;
66+ }
67+
68+ function loopPerfCommandProjection (
69+ command : string | null | undefined ,
70+ ) : Partial < Record < "command" , string > > {
71+ return optional ( "command" , sanitizeLoopPerfCommand ( command ) ) ;
72+ }
73+
74+ function sanitizeLoopPerfRawBytes ( rawBytes : string ) : string {
75+ return truncateUtf8 (
76+ redactCredentialLikeText ( stripUnsafeText ( rawBytes ) ) ,
77+ LOOP_PERF_PARSE_FAILURE_RAW_BYTES_MAX_BYTES ,
78+ ) ;
79+ }
80+
81+ function sanitizeLoopPerfErrorMessage ( errorMessage : string ) : string {
82+ return truncateUtf8 (
83+ redactCredentialLikeText ( stripUnsafeText ( errorMessage ) ) ,
84+ LOOP_PERF_PARSE_FAILURE_ERROR_MESSAGE_MAX_BYTES ,
85+ ) ;
86+ }
87+
2188/**
2289 * Capture the current byte-offset of `perf.jsonl` before the orchestrator
2390 * spawns. This is the initial high-water mark (HWM) used by the streaming
@@ -116,6 +183,7 @@ const iterationSchema = z.object({
116183 event : z . literal ( "iteration" ) ,
117184 run_id : z . string ( ) ,
118185 iteration : z . number ( ) . int ( ) ,
186+ command : z . string ( ) . nullish ( ) ,
119187 started_at : z . string ( ) ,
120188 ended_at : z . string ( ) ,
121189 duration_s : z . number ( ) ,
@@ -127,6 +195,7 @@ const pipelineStepSchema = z.object({
127195 event : z . literal ( "pipeline_step" ) ,
128196 run_id : z . string ( ) ,
129197 iteration : z . number ( ) . int ( ) ,
198+ command : z . string ( ) . nullish ( ) ,
130199 // The producer emits non-integer step numbers (e.g. 8.5 for
131200 // write_merged_patterns) to slot synthetic sub-steps between the integer
132201 // pipeline positions. LoopPerfPipelineStepEvent.step is already typed as
@@ -351,7 +420,7 @@ function toLoopPerfDiagnostics(
351420 event : "run" ,
352421 runId : raw . run_id ,
353422 startedAt : raw . started_at ,
354- ...optional ( "command" , raw . command ) ,
423+ ...loopPerfCommandProjection ( raw . command ) ,
355424 ...optional ( "repo" , raw . repo ) ,
356425 ...optional ( "branch" , raw . branch ) ,
357426 } ;
@@ -364,7 +433,7 @@ function toLoopPerfDiagnostics(
364433 status : raw . status ,
365434 startedAt : raw . started_at ,
366435 ...optional ( "startSha" , raw . start_sha ) ,
367- ...optional ( "command" , raw . command ) ,
436+ ...loopPerfCommandProjection ( raw . command ) ,
368437 } ;
369438 case "iteration" :
370439 return {
@@ -375,6 +444,7 @@ function toLoopPerfDiagnostics(
375444 endedAt : raw . ended_at ,
376445 durationS : raw . duration_s ,
377446 status : raw . status ,
447+ ...loopPerfCommandProjection ( raw . command ) ,
378448 ...optional ( "claudeExitCode" , raw . claude_exit_code ) ,
379449 } ;
380450 case "pipeline_step" :
@@ -388,6 +458,7 @@ function toLoopPerfDiagnostics(
388458 endedAt : raw . ended_at ,
389459 durationS : raw . duration_s ,
390460 skipped : raw . skipped ,
461+ ...loopPerfCommandProjection ( raw . command ) ,
391462 ...optional ( "exitCode" , raw . exit_code ) ,
392463 } ;
393464 case "agent" :
@@ -401,7 +472,7 @@ function toLoopPerfDiagnostics(
401472 startedAt : raw . started_at ,
402473 endedAt : raw . ended_at ,
403474 durationS : raw . duration_s ,
404- ...optional ( "command" , raw . command ) ,
475+ ...loopPerfCommandProjection ( raw . command ) ,
405476 ...optional ( "model" , raw . model ) ,
406477 ...optional ( "parentSessionId" , raw . parent_session_id ) ,
407478 ...optional ( "inputTokens" , raw . input_tokens ) ,
@@ -419,7 +490,7 @@ function toLoopPerfDiagnostics(
419490 agentId : raw . agent_id ,
420491 toolName : raw . tool_name ,
421492 startedAt : raw . started_at ,
422- ...optional ( "command" , raw . command ) ,
493+ ...loopPerfCommandProjection ( raw . command ) ,
423494 ...optional ( "endedAt" , raw . ended_at ) ,
424495 ...optional ( "durationS" , raw . duration_s ) ,
425496 ...optional ( "ok" , raw . ok ) ,
@@ -437,7 +508,7 @@ function toLoopPerfDiagnostics(
437508 endedAt : raw . ended_at ,
438509 durationS : raw . duration_s ,
439510 ok : raw . ok ,
440- ...optional ( "command" , raw . command ) ,
511+ ...loopPerfCommandProjection ( raw . command ) ,
441512 ...optional ( "phase" , attributedPhase ) ,
442513 } ;
443514 case "spawn" :
@@ -447,7 +518,7 @@ function toLoopPerfDiagnostics(
447518 iteration : raw . iteration ,
448519 parentAgentId : raw . parent_agent_id ,
449520 startedAt : raw . started_at ,
450- ...optional ( "command" , raw . command ) ,
521+ ...loopPerfCommandProjection ( raw . command ) ,
451522 ...optional ( "parentSessionId" , raw . parent_session_id ) ,
452523 ...optional ( "plannedSubagentType" , raw . planned_subagent_type ) ,
453524 ...optional ( "phase" , attributedPhase ) ,
@@ -479,6 +550,31 @@ function eventToCategory(event: RawPerfEvent["event"]): TelemetryCategory {
479550 return PERF_EVENT_CATEGORIES [ event ] ;
480551}
481552
553+ function emitLoopPerfParseFailure (
554+ ctx : ParseChunkContext ,
555+ options : {
556+ message : string ;
557+ lineNumber : number ;
558+ rawBytes : string ;
559+ errorMessage : string ;
560+ } ,
561+ ) : void {
562+ ctx . telemetryEmitter . emit ( {
563+ severity : "warn" ,
564+ category : "loop.perf.parse_failure" ,
565+ message : options . message ,
566+ trace : ctx . traceContext ,
567+ diagnostics : {
568+ loopPerf : {
569+ event : "parse_failure" ,
570+ lineNumber : options . lineNumber ,
571+ rawBytes : sanitizeLoopPerfRawBytes ( options . rawBytes ) ,
572+ errorMessage : sanitizeLoopPerfErrorMessage ( options . errorMessage ) ,
573+ } ,
574+ } ,
575+ } ) ;
576+ }
577+
482578// ---------------------------------------------------------------------------
483579// parseAndEmitChunk — the main streaming helper
484580// ---------------------------------------------------------------------------
@@ -521,6 +617,8 @@ export function parseAndEmitChunk(
521617 const completeLines = parts . slice ( 0 , - 1 ) ;
522618
523619 let lineNumber = ctx . lineNumberBase ;
620+ let parseFailureEventsEmitted = 0 ;
621+ let parseFailuresSuppressed = 0 ;
524622
525623 for ( const rawLine of completeLines ) {
526624 lineNumber += 1 ;
@@ -538,20 +636,17 @@ export function parseAndEmitChunk(
538636 try {
539637 parsedJson = JSON . parse ( line ) ;
540638 } catch ( err ) {
541- ctx . telemetryEmitter . emit ( {
542- severity : "warn" ,
543- category : "loop.perf.parse_failure" ,
544- message : "perf.jsonl: malformed JSON line" ,
545- trace : ctx . traceContext ,
546- diagnostics : {
547- loopPerf : {
548- event : "parse_failure" ,
549- lineNumber,
550- rawBytes : line ,
551- errorMessage : err instanceof Error ? err . message : String ( err ) ,
552- } ,
553- } ,
554- } ) ;
639+ if ( parseFailureEventsEmitted < LOOP_PERF_PARSE_FAILURE_MAX_EVENTS_PER_CHUNK ) {
640+ emitLoopPerfParseFailure ( ctx , {
641+ message : "perf.jsonl: malformed JSON line" ,
642+ lineNumber,
643+ rawBytes : line ,
644+ errorMessage : err instanceof Error ? err . message : String ( err ) ,
645+ } ) ;
646+ parseFailureEventsEmitted += 1 ;
647+ } else {
648+ parseFailuresSuppressed += 1 ;
649+ }
555650 continue ;
556651 }
557652
@@ -574,20 +669,17 @@ export function parseAndEmitChunk(
574669 // --- Zod validation ---
575670 const parsed = perfEventSchema . safeParse ( parsedJson ) ;
576671 if ( ! parsed . success ) {
577- ctx . telemetryEmitter . emit ( {
578- severity : "warn" ,
579- category : "loop.perf.parse_failure" ,
580- message : "perf.jsonl: Zod validation failure" ,
581- trace : ctx . traceContext ,
582- diagnostics : {
583- loopPerf : {
584- event : "parse_failure" ,
585- lineNumber,
586- rawBytes : line ,
587- errorMessage : parsed . error . message ,
588- } ,
589- } ,
590- } ) ;
672+ if ( parseFailureEventsEmitted < LOOP_PERF_PARSE_FAILURE_MAX_EVENTS_PER_CHUNK ) {
673+ emitLoopPerfParseFailure ( ctx , {
674+ message : "perf.jsonl: Zod validation failure" ,
675+ lineNumber,
676+ rawBytes : line ,
677+ errorMessage : parsed . error . message ,
678+ } ) ;
679+ parseFailureEventsEmitted += 1 ;
680+ } else {
681+ parseFailuresSuppressed += 1 ;
682+ }
591683 continue ;
592684 }
593685
@@ -621,6 +713,15 @@ export function parseAndEmitChunk(
621713 } ) ;
622714 }
623715
716+ if ( parseFailuresSuppressed > 0 ) {
717+ emitLoopPerfParseFailure ( ctx , {
718+ message : "perf.jsonl: parse failures suppressed" ,
719+ lineNumber,
720+ rawBytes : "" ,
721+ errorMessage : `${ parseFailuresSuppressed } additional parse failure event(s) suppressed in this chunk` ,
722+ } ) ;
723+ }
724+
624725 return {
625726 newPriorLineBuffer,
626727 newLineNumberBase : lineNumber ,
@@ -753,7 +854,9 @@ export function startLoopPerfTelemetryWatcher(
753854 event : "parse_failure" ,
754855 lineNumber : lineNumberBase ,
755856 rawBytes : "" ,
756- errorMessage : err instanceof Error ? err . message : String ( err ) ,
857+ errorMessage : sanitizeLoopPerfErrorMessage (
858+ err instanceof Error ? err . message : String ( err ) ,
859+ ) ,
757860 } ,
758861 } ,
759862 } ) ;
@@ -810,7 +913,9 @@ export function startLoopPerfTelemetryWatcher(
810913 event : "parse_failure" ,
811914 lineNumber : lineNumberBase ,
812915 rawBytes : "" ,
813- errorMessage : err instanceof Error ? err . message : String ( err ) ,
916+ errorMessage : sanitizeLoopPerfErrorMessage (
917+ err instanceof Error ? err . message : String ( err ) ,
918+ ) ,
814919 } ,
815920 } ,
816921 } ) ;
@@ -842,7 +947,9 @@ export function startLoopPerfTelemetryWatcher(
842947 event : "parse_failure" ,
843948 lineNumber : 0 ,
844949 rawBytes : "" ,
845- errorMessage : err instanceof Error ? err . message : String ( err ) ,
950+ errorMessage : sanitizeLoopPerfErrorMessage (
951+ err instanceof Error ? err . message : String ( err ) ,
952+ ) ,
846953 } ,
847954 } ,
848955 } ) ;
@@ -1032,7 +1139,9 @@ export function reconcileLoopPerfTelemetry(
10321139 event : "parse_failure" ,
10331140 lineNumber : 0 ,
10341141 rawBytes : "" ,
1035- errorMessage : err instanceof Error ? err . message : String ( err ) ,
1142+ errorMessage : sanitizeLoopPerfErrorMessage (
1143+ err instanceof Error ? err . message : String ( err ) ,
1144+ ) ,
10361145 } ,
10371146 } ,
10381147 } ) ;
@@ -1095,9 +1204,7 @@ export function reconcileLoopPerfTelemetry(
10951204 toolName : sentinel . tool_name ,
10961205 startedAt : sentinel . started_at ,
10971206 iteration : sentinel . iteration ,
1098- ...( sentinel . command !== null && sentinel . command !== undefined
1099- ? { command : sentinel . command }
1100- : { } ) ,
1207+ ...loopPerfCommandProjection ( sentinel . command ) ,
11011208 endedAt : null ,
11021209 durationS : null ,
11031210 ok : null ,
@@ -1129,7 +1236,9 @@ export function reconcileLoopPerfTelemetry(
11291236 event : "parse_failure" ,
11301237 lineNumber : 0 ,
11311238 rawBytes : "" ,
1132- errorMessage : err instanceof Error ? err . message : String ( err ) ,
1239+ errorMessage : sanitizeLoopPerfErrorMessage (
1240+ err instanceof Error ? err . message : String ( err ) ,
1241+ ) ,
11331242 } ,
11341243 } ,
11351244 } ) ;
0 commit comments