Skip to content

Commit a97db88

Browse files
authored
fix(redaction): redact credentials whose trigger a greedy span swallowed (#1158)
Two credentials glued end to end share one recovering property: the second one's trigger prefix is where redaction can re-anchor. For fixed-length bodies that anchor is exact (AKIA{16} stops at the next key), which is the case the anchored end-chaining always covered. For unbounded bodies it isn't: github's [A-Za-z0-9]{36,} body eats the literal "ghp" of the token glued behind it, so the first match ends PAST the second trigger, the scan that re-anchors at the span end finds nothing there, and the second credential leaves the output as +_bbbb... / .eyJzdWIi... plaintext: RedactString("ghp_aaa..36a..ghp_bbb..36b..") -> "[REDACTED]_bbbb......" (second token's body printed in full) RedactString(jwtA+jwtB) -> "[REDACTED].eyJzdWIiOiI5ODc2...SA" (second token's payload+sig) The same holds across shapes (a github key directly feeding "sk-ant-..." through its body) and for three tokens in one run. redactAdjacent now follows the run a match starts: at each boundary it re-binds the match at the EARLIEST trigger occurrence inside the span that stills validates as its shape (the head before the trigger must match the anchored pattern exactly, and the run chained from the trigger must reach past the span, so a mid-body lookalike and a mid-span cut that recovers nothing leave the span alone), and continues under whichever shape matched the swallowed trigger. Candidate examination is capped at maxTriggerOccurrences per span so body spam of trigger lookalikes costs what the span costs, and the first validating cut wins. End-chaining for fixed-length bodies, single-credential shapes, custom replacement templates, and the split path's announced no-op-for-contiguous text guarantee are all unchanged; the regression tests pin the glued pairs that previously leaked plus the AWS chaining and lone-token cases.
1 parent ef4da13 commit a97db88

2 files changed

Lines changed: 411 additions & 30 deletions

File tree

‎internal/redaction/redaction.go‎

Lines changed: 240 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,42 @@ var textSecretPrefixes = [][]string{
112112
{"eyJ"},
113113
}
114114

115+
type secretTrigger struct {
116+
literal string
117+
// shape indexes the anchoredSecretPatterns entry the trigger belongs to,
118+
// so a cut can chain the bytes after it against the shape the trigger
119+
// actually starts.
120+
shape int
121+
}
122+
123+
// secretTriggers flattens textSecretPrefixes into one sorted list for the
124+
// swallowed-trigger search below. Sorted so cuts are discovered earliest-first
125+
// regardless of which pattern list they came from.
126+
var secretTriggers = func() []secretTrigger {
127+
pairs := []secretTrigger{}
128+
for shape, prefixes := range textSecretPrefixes {
129+
for _, prefix := range prefixes {
130+
pairs = append(pairs, secretTrigger{literal: prefix, shape: shape})
131+
}
132+
}
133+
sort.Slice(pairs, func(i, j int) bool { return pairs[i].literal < pairs[j].literal })
134+
return pairs
135+
}()
136+
137+
// longestTrigger bounds how far past a swallowed-scored span the trigger
138+
// inspection may need to look: an occurrence whose last bytes run past the
139+
// span end (a greedy body stops at a byte outside its class, and several
140+
// triggers end in one).
141+
var longestTrigger = func() int {
142+
longest := 0
143+
for _, trigger := range secretTriggers {
144+
if len(trigger.literal) > longest {
145+
longest = len(trigger.literal)
146+
}
147+
}
148+
return longest
149+
}()
150+
115151
// anchoredSecretPatterns mirrors textSecretPatterns with the leading \b
116152
// replaced by \A, so each shape can be tested at an exact byte offset. A
117153
// credential that starts exactly where a redacted span ends (two AWS keys
@@ -129,52 +165,226 @@ var anchoredSecretPatterns = func() []*regexp.Regexp {
129165
return out
130166
}()
131167

132-
// redactAdjacent redacts every match of pattern, plus any match of anchored
133-
// that begins exactly where a redacted span ends. This catches credentials
134-
// glued end to end (AKIA...AKIA...) without weakening the leading-boundary
135-
// rule elsewhere: a mid-word occurrence that does not abut a redacted span
136-
// still does not match. Chained runs (three or more glued credentials) are
137-
// followed to the end.
168+
// A credential's body can run into the trigger prefix of the NEXT one when
169+
// both are glued end to end. For fixed-length bodies this cannot happen — the
170+
// `{16}` stops the body exactly where the next AKIA...AKIA... key begins,
171+
// which is the case the anchored end-chaining always covered. An unbounded
172+
// body ({36,} on a github key, {20,} on an API key) swallows whichever trigger
173+
// bytes sit inside its class — "ghp" after a github key is all lowercase
174+
// letters — so the first match ends PAST the next credential's trigger, and a
175+
// scan that re-anchors at the span end finds nothing: the second credential
176+
// is never redacted.
177+
//
178+
// redactedSpan is one region of a redaction: the slice s[start:end] and the
179+
// bytes to write in its place. Chained runs can link shapes together, so a
180+
// span says which shape produced it rather than assuming the caller's pattern
181+
// covered it.
182+
type redactedSpan struct {
183+
start int
184+
end int
185+
// text replaces s[start:end]. Every entry in textSecretPatterns captures
186+
// nothing, so this is the replacement itself unless a custom replacement
187+
// template references a group; anchoring the expansion to the shape that
188+
// matched keeps that working for cross-shape chains.
189+
text string
190+
}
191+
192+
// redactAdjacent redacts every match of pattern, plus every credential glued
193+
// into the runs those matches start. Runs follow three loss-of-boundary
194+
// mechanisms: end-chaining (a credential glued exactly at the previous span's
195+
// end, the AKIA...AKIA... case), all-shape chaining (a glued credential of a
196+
// different shape), and the swallowed-trigger cut (one credential's greedy
197+
// body runs into the next one's trigger prefix, hiding it from any scan that
198+
// re-anchors at a span end). A run ends at the first byte boundary where no
199+
// shape matches; a mid-word lookalike never matches and never starts one.
138200
func redactAdjacent(s string, pattern, anchored *regexp.Regexp, replacement string) string {
139-
spans := pattern.FindAllStringIndex(s, -1)
140-
// Bound the outer loop to the original match count. The inner loop
141-
// appends chained spans to the same slice; re-reading len(spans) would
142-
// re-chain every appended span, growing the span count (and the
143-
// FindStringIndex calls) exponentially on long glued runs.
144-
n := len(spans)
201+
plain := pattern.FindAllStringIndex(s, -1)
202+
if len(plain) == 0 {
203+
return s
204+
}
205+
// Bound the outer loop to the original match count. Every span collects
206+
// its own run; re-reading len(plain) after appends would re-run every
207+
// appended credential and grow the span count (with the anchored scans
208+
// each drags) exponentially on long glued runs.
209+
n := len(plain)
210+
spans := make([]redactedSpan, 0, n+4)
145211
for i := 0; i < n; i++ {
146-
end := spans[i][1]
147-
for end < len(s) {
148-
loc := anchored.FindStringIndex(s[end:])
149-
if loc == nil || loc[0] != 0 || loc[1] <= 0 {
150-
break
151-
}
152-
newEnd := end + loc[1]
153-
spans = append(spans, []int{end, newEnd})
154-
end = newEnd
155-
}
212+
spans = appendCredentialRun(s, plain[i][0], plain[i][1], replacement, spans)
156213
}
214+
return writeRedactedSpans(s, spans)
215+
}
216+
217+
func writeRedactedSpans(s string, spans []redactedSpan) string {
157218
if len(spans) == 0 {
158219
return s
159220
}
160-
sort.Slice(spans, func(i, j int) bool { return spans[i][0] < spans[j][0] })
221+
sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start })
161222
var out strings.Builder
162223
pos := 0
163224
for _, sp := range spans {
164-
if sp[0] < pos {
225+
if sp.start < pos {
165226
continue // overlapping span, already covered
166227
}
167-
out.WriteString(s[pos:sp[0]])
168-
// s[sp[0]:sp[1]] is a match (anchored spans match at \b when taken
169-
// alone), so ReplaceAllString expands $ references exactly as the
170-
// plain loop below would.
171-
out.WriteString(pattern.ReplaceAllString(s[sp[0]:sp[1]], replacement))
172-
pos = sp[1]
228+
out.WriteString(s[pos:sp.start])
229+
out.WriteString(sp.text)
230+
pos = sp.end
173231
}
174232
out.WriteString(s[pos:])
175233
return out.String()
176234
}
177235

236+
// appendCredentialRun follows the path that begins exactly at start, whose
237+
// first plain match ended at end. Every step finds the shape that matches at
238+
// the current boundary, tries to re-bind that match at a swallowed trigger,
239+
// and otherwise consumes the whole match; the run ends where no shape does.
240+
func appendCredentialRun(s string, start, end int, replacement string, spans []redactedSpan) []redactedSpan {
241+
cur := start
242+
for cur < len(s) {
243+
shape, matchEnd := firstShapeAt(s, cur)
244+
if shape < 0 || matchEnd == cur {
245+
break
246+
}
247+
if cutStart, cut := earliestSwallowedCut(s, cur, matchEnd, shape); cut {
248+
// The bytes before the trigger are the credential that matched;
249+
// from the trigger on the run continues under the next shape.
250+
spans = append(spans, redactedSpan{start: cur, end: cutStart, text: shapeReplacement(shape, s[cur:cutStart], replacement)})
251+
cur = cutStart
252+
continue
253+
}
254+
spans = append(spans, redactedSpan{start: cur, end: matchEnd, text: shapeReplacement(shape, s[cur:matchEnd], replacement)})
255+
cur = matchEnd
256+
}
257+
return spans
258+
}
259+
260+
func shapeReplacement(shape int, slice, replacement string) string {
261+
return anchoredSecretPatterns[shape].ReplaceAllString(slice, replacement)
262+
}
263+
264+
// firstShapeAt returns the first anchored shape that matches exactly at off,
265+
// with the end of its match. Shapes are tried in declaration order, so the
266+
// strict JWT form wins over the loose one; both redact the same bytes, so the
267+
// tie-break only decides which regex expanded the replacement.
268+
func firstShapeAt(s string, off int) (shape int, end int) {
269+
for shape, anchored := range anchoredSecretPatterns {
270+
loc := anchored.FindStringIndex(s[off:])
271+
if loc != nil && loc[0] == 0 && loc[1] > 0 {
272+
return shape, off + loc[1]
273+
}
274+
}
275+
return -1, off
276+
}
277+
278+
// earliestSwallowedCut searches the matched span [start, end) of shape for a
279+
// trigger literal that begins a valid credential right there: the head (the
280+
// bytes before it) still matches the shape as a whole, and the run chained
281+
// from the trigger reaches past the span. The earliest such trigger is the
282+
// cut a boundary at that byte would have produced.
283+
//
284+
// The head requirement and the past-the-end requirement keep the cut honest:
285+
// a trigger spelled mid-body that does not begin a credential leaves the span
286+
// alone, and a cut that stayed inside the original span recovers no bytes —
287+
// it would only split one redaction in two.
288+
//
289+
// The candidate set is the trigger occurrences inside the span, bounded by
290+
// the span plus one trigger length; every occurrence of every trigger in that
291+
// window is a candidate, earliest-first.
292+
func earliestSwallowedCut(s string, start, end, shape int) (int, bool) {
293+
if end-start <= 1 {
294+
return 0, false
295+
}
296+
lookahead := end + longestTrigger
297+
if lookahead > len(s) {
298+
lookahead = len(s)
299+
}
300+
window := s[start+1 : lookahead]
301+
302+
// A trigger occurrence whose bytes begin inside the span may extend past
303+
// its end: the greedy body stops at one byte outside its class, and
304+
// several triggers end in exactly such a byte ("ghp_"'s underscore,
305+
// "sk-ant-" plus a hyphen).
306+
anchored := anchoredSecretPatterns[shape]
307+
examined := 0
308+
for _, q := range triggerOccurrences(window, start+1, end) {
309+
if examined >= maxTriggerOccurrences {
310+
// Occurrences are earliest-first, so the ones past the cap are the
311+
// farthest into the span; a cut among them is missed and the
312+
// greedy span stands, as it did before the cut existed.
313+
break
314+
}
315+
examined++
316+
// The bytes before the trigger must still be a complete match of this
317+
// shape: a trigger spelling inside a body whose head does not
318+
// validate is prose-shaped, not a swallowed credential.
319+
loc := anchored.FindStringIndex(s[start:q])
320+
if loc == nil || loc[0] != 0 || loc[1] != q-start {
321+
continue
322+
}
323+
if runReach(s, q) > end {
324+
return q, true
325+
}
326+
}
327+
return 0, false
328+
}
329+
330+
// maxTriggerOccurrences bounds how many trigger occurrences one span is
331+
// examined for, after sorting. Real glued credentials announce their trigger
332+
// within reach of the span's own start, so a distant lookalike run does not
333+
// get walked to its end; the split path makes the same trade ("at most two
334+
// passes and no more"). A cut found past the cap is missed and the greedy
335+
// span stands, as it did before the cut existed.
336+
const maxTriggerOccurrences = 64
337+
338+
// triggerOccurrences returns every occurrence of every trigger literal that
339+
// begins inside [windowStart, windowEnd), where window is the slice rooted at
340+
// windowStart that the caller bounded so occurrences could reach one trigger
341+
// length past its end. Earliest first, duplicates collapsed.
342+
func triggerOccurrences(window string, windowStart, windowEnd int) []int {
343+
occurrences := make([]int, 0, 4)
344+
for _, trigger := range secretTriggers {
345+
at := 0
346+
for {
347+
found := strings.Index(window[at:], trigger.literal)
348+
if found < 0 {
349+
break
350+
}
351+
q := windowStart + at + found
352+
if q >= windowEnd {
353+
break // later occurrences for this trigger start even later
354+
}
355+
occurrences = append(occurrences, q)
356+
at += found + 1
357+
}
358+
}
359+
sort.Ints(occurrences)
360+
// In-place deduplicate: duplicates come only from triggers shared between
361+
// shapes ("eyJ"), and zero is never an occurrence — the scan starts past it.
362+
unique := occurrences[:0]
363+
for i, q := range occurrences {
364+
if i > 0 && q == occurrences[i-1] {
365+
continue
366+
}
367+
unique = append(unique, q)
368+
}
369+
return unique
370+
}
371+
372+
// runReach reports how far the credential run beginning exactly at off
373+
// extends: each boundary picks the first shape that matches there, until one
374+
// does not. The run is followed, not redacted; this is the shared-size check
375+
// that keeps a cut honest.
376+
func runReach(s string, off int) int {
377+
cur := off
378+
for cur < len(s) {
379+
_, matchEnd := firstShapeAt(s, cur)
380+
if matchEnd == cur {
381+
break
382+
}
383+
cur = matchEnd
384+
}
385+
return cur
386+
}
387+
178388
var (
179389
privateKeyPattern = regexp.MustCompile(`(?s)-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----`)
180390
jsonStringPattern = regexp.MustCompile(`("([^"\\]*(?:\\.[^"\\]*)*)"\s*:\s*)"([^"\\]*(?:\\.[^"\\]*)*)"`)

0 commit comments

Comments
 (0)