88import platform
99import re
1010import subprocess
11+ from datetime import datetime
1112from pathlib import Path
1213from typing import Any , BinaryIO
1314from urllib .parse import quote , quote_plus
1718
1819REDACTED = "[REDACTED]"
1920_SECRET_KEY = re .compile (
20- r"password|passwd|secret|token$|api[_-]?key|access[_-]?key|private[_-]?key|"
21+ r"password|passwd|secret|token$|api[_-]?key|access[_-]?key|private[_-]?key|aes[_-]?key| "
2122 r"authorization|cookie|credential|(?:client|app|signing|encrypt(?:ion)?)[_-]?key|"
2223 r"recovery[_-]?code|(?:^|\.)key$|^headers$|^env$" ,
2324 re .IGNORECASE ,
2627_AUTH = re .compile (r"\b(Bearer|Basic|ApiKey)\s+[^\s\"'<>]+" , re .IGNORECASE )
2728_COOKIE = re .compile (r"(\b(?:set-cookie|cookie)\s*:\s*)[^\r\n]+" , re .IGNORECASE )
2829_URL_AUTH = re .compile (r"(https?://)[^/\s@]+@" , re .IGNORECASE )
29- _ASSIGNMENT = re .compile (
30- r"((?:[\w.-]*(?:token|secret|password|passwd|api[_-]?key|access[_-]?key|"
31- r"(?:client|app|signing|encrypt(?:ion)?) [_-]?key|recovery [_-]?code) "
32- r"[\w.-]*|authorization|cookie|\bkey)[\"']?\s*[:=]\s*) "
33- r"(\"[^\"]*\"|'[^']*'|[^\s&,;}]+) " ,
30+ _ASSIGNMENT = re .compile (r"(?<![\w.-])([\w.-]+)[\"']?\s*[:=]\s*" )
31+ _ASSIGNMENT_KEY = re . compile (
32+ r"token|secret|password|passwd|api[_-]?key|access [_-]?key|aes [_-]?key| "
33+ r"(?:client|app|signing|encrypt(?:ion)?)[_-]?key|recovery[_-]?code| "
34+ r"(?:authorization|cookie|\bkey)$ " ,
3435 re .IGNORECASE ,
3536)
37+ _ASSIGNMENT_VALUE = re .compile (r"\"[^\"]*\"|'[^']*'|[^\s&,;}]+" )
3638_TOKEN = re .compile (
3739 r"\b(?:sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9_]{20,}|"
3840 r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})"
4951 r")(?!\w|\.\w)"
5052)
5153MAX_LOG_LINE_BYTES = 1024 * 1024
54+ _LOG_TIMESTAMP = re .compile (
55+ r"^\[?(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}"
56+ r"(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?)(?:\]|\s|$)"
57+ )
5258
5359
5460class DiagnosticRedactor :
@@ -57,6 +63,7 @@ class DiagnosticRedactor:
5763 def __init__ (self , * , enabled : bool = True ) -> None :
5864 self .enabled = enabled
5965 self .secrets : set [str ] = set ()
66+ self ._sorted_secrets : list [str ] | None = None
6067 self .public_ips : dict [str , str ] = {}
6168
6269 def collect (self , value : Any , sensitive : bool = False ) -> None :
@@ -79,6 +86,7 @@ def collect(self, value: Any, sensitive: bool = False) -> None:
7986 self .secrets .update (
8087 (text , quote (text , safe = "" ), quote_plus (text ), json .dumps (text )[1 :- 1 ])
8188 )
89+ self ._sorted_secrets = None
8290
8391 def redact (self , value : Any , sensitive : bool = False ) -> Any :
8492 """Return a sanitized copy while retaining containers and boolean flags.
@@ -118,15 +126,59 @@ def text(self, value: str) -> str:
118126 """
119127 if not self .enabled :
120128 return value
121- value = _ANSI .sub ("" , value )
122- value = _PRIVATE_KEY .sub (REDACTED , value )
123- for secret in sorted (self .secrets , key = len , reverse = True ):
124- value = value .replace (secret , REDACTED )
125- value = _AUTH .sub (lambda match : f"{ match [1 ]} { REDACTED } " , value )
126- value = _COOKIE .sub (lambda match : f"{ match [1 ]} { REDACTED } " , value )
127- value = _URL_AUTH .sub (lambda match : f"{ match [1 ]} { REDACTED } @" , value )
128- value = _ASSIGNMENT .sub (lambda match : f"{ match [1 ]} { REDACTED } " , value )
129- value = _TOKEN .sub (REDACTED , value )
129+ if "\x1b [" in value :
130+ value = _ANSI .sub ("" , value )
131+ if "PRIVATE KEY-----" in value :
132+ value = _PRIVATE_KEY .sub (REDACTED , value )
133+ if self .secrets :
134+ if self ._sorted_secrets is None :
135+ self ._sorted_secrets = sorted (self .secrets , key = len , reverse = True )
136+ for secret in self ._sorted_secrets :
137+ value = value .replace (secret , REDACTED )
138+ # These cheap substring checks avoid running every regex on ordinary
139+ # ASCII log lines. Unicode case-insensitive regex matches can include
140+ # characters that lower() does not normalize to the ASCII hint.
141+ lowered = value .lower ()
142+ ascii_only = value .isascii ()
143+ if not ascii_only or any (
144+ prefix in lowered for prefix in ("bearer" , "basic" , "apikey" )
145+ ):
146+ value = _AUTH .sub (lambda match : f"{ match [1 ]} { REDACTED } " , value )
147+ if not ascii_only or "cookie" in lowered :
148+ value = _COOKIE .sub (lambda match : f"{ match [1 ]} { REDACTED } " , value )
149+ if "://" in value and "@" in value :
150+ value = _URL_AUTH .sub (lambda match : f"{ match [1 ]} { REDACTED } @" , value )
151+ if not ascii_only or any (
152+ hint in lowered
153+ for hint in (
154+ "token" ,
155+ "secret" ,
156+ "password" ,
157+ "passwd" ,
158+ "key" ,
159+ "code" ,
160+ "authorization" ,
161+ "cookie" ,
162+ )
163+ ):
164+ # Scan assignment prefixes without consuming ordinary values, which
165+ # may themselves contain a nested credential assignment.
166+ parts = []
167+ end = 0
168+ for match in _ASSIGNMENT .finditer (value ):
169+ if match .start () < end or not _ASSIGNMENT_KEY .search (match [1 ]):
170+ continue
171+ assigned = _ASSIGNMENT_VALUE .match (value , match .end ())
172+ if assigned is not None :
173+ parts .extend ((value [end : match .end ()], REDACTED ))
174+ end = assigned .end ()
175+ parts .append (value [end :])
176+ value = "" .join (parts )
177+ if any (
178+ prefix in value
179+ for prefix in ("sk-" , "ghp_" , "gho_" , "ghu_" , "ghs_" , "ghr_" , "eyJ" )
180+ ):
181+ value = _TOKEN .sub (REDACTED , value )
130182 parts = []
131183 end = 0
132184 for match in _IP_LITERAL .finditer (value ):
@@ -136,6 +188,12 @@ def text(self, value: str) -> str:
136188 literal [1 :- 1 ] if literal .startswith ("[" ) else literal .rstrip ("." )
137189 )
138190 host , separator , scope = candidate .partition ("%" )
191+ if (
192+ ":" in host
193+ and "::" not in host
194+ and host .count (":" ) != (6 if "." in host else 7 )
195+ ):
196+ continue
139197 try :
140198 address = ipaddress .ip_address (host )
141199 except ValueError :
@@ -156,18 +214,29 @@ def text(self, value: str) -> str:
156214 parts .append (value [end :])
157215 return "" .join (parts )
158216
159- def copy_log (self , source : BinaryIO , target : BinaryIO , limit : int ) -> bool :
217+ def copy_log (
218+ self ,
219+ source : BinaryIO ,
220+ target : BinaryIO ,
221+ limit : int ,
222+ * ,
223+ start_time : int | None = None ,
224+ end_time : int | None = None ,
225+ ) -> bool :
160226 """Stream complete text lines, skipping oversized lines and private keys.
161227
162228 Args:
163229 source: An already verified regular file, positioned at its start.
164230 target: Open ZIP entry for redacted text.
165231 limit: Maximum source bytes to consume, fixed before reading.
232+ start_time: Inclusive minimum record time, in Unix seconds.
233+ end_time: Inclusive maximum record time, in Unix seconds.
166234
167235 Returns:
168236 Whether content was omitted to respect line or file limits.
169237 """
170- if not self .enabled :
238+ filtering = start_time is not None or end_time is not None
239+ if not self .enabled and not filtering :
171240 remaining = limit
172241 while remaining :
173242 chunk = source .read (min (remaining , MAX_LOG_LINE_BYTES ))
@@ -179,42 +248,96 @@ def copy_log(self, source: BinaryIO, target: BinaryIO, limit: int) -> bool:
179248 truncated = False
180249 oversized = False
181250 private_key = False
251+ pending : list [str ] = []
252+ pending_bytes = 0
182253 remaining = limit
254+ include_record = not filtering
183255 while remaining :
184256 line = source .readline (min (remaining , MAX_LOG_LINE_BYTES + 1 ))
185257 if not line :
186258 break
187259 remaining -= len (line )
260+ if filtering and not oversized and not private_key :
261+ # Timestamp-free continuation lines inherit their record's
262+ # selection; content before the first dated record is excluded.
263+ prefix = line [:160 ].decode ("utf-8" , errors = "replace" )
264+ match = _LOG_TIMESTAMP .match (_ANSI .sub ("" , prefix ))
265+ timestamp = None
266+ dated_record = match is not None
267+ try :
268+ if match is not None :
269+ timestamp = datetime .fromisoformat (
270+ match [1 ].replace ("Z" , "+00:00" ).replace ("," , "." )
271+ ).timestamp ()
272+ elif prefix .startswith ("{" ):
273+ record = json .loads (line )
274+ if isinstance (record , dict ):
275+ dated_record = True
276+ if "time_unix_nano" in record :
277+ timestamp = int (record ["time_unix_nano" ]) / 1e9
278+ elif isinstance (record .get ("time" ), (int , float )):
279+ timestamp = record ["time" ]
280+ elif "start_time_unix_nano" in record :
281+ timestamp = int (record ["start_time_unix_nano" ]) / 1e9
282+ except (ValueError , TypeError , OverflowError , OSError ):
283+ pass
284+ if dated_record :
285+ include_record = (
286+ timestamp is not None
287+ and (start_time is None or timestamp >= start_time )
288+ and (end_time is None or timestamp <= end_time )
289+ )
188290 complete = line .endswith (b"\n " ) or len (line ) <= MAX_LOG_LINE_BYTES
189291 if oversized or not complete :
190292 oversized = not line .endswith (b"\n " )
191- truncated = True
192- continue
293+ if self .enabled :
294+ truncated = True
295+ continue
193296 # Never emit a fragment cut at the export boundary: it could be a token.
194297 if not remaining and not line .endswith (b"\n " ) and source .read (1 ):
195298 truncated = True
196299 break
300+ if not self .enabled :
301+ if include_record :
302+ target .write (line )
303+ continue
197304 text = line .decode ("utf-8" , errors = "replace" )
198305 if "-----BEGIN " in text and "PRIVATE KEY-----" in text :
199- private_key = True
200- target . write (( REDACTED + "\n " ). encode ())
201- if private_key :
306+ private_key = not ( "-----END " in text and "PRIVATE KEY-----" in text )
307+ text = REDACTED + "\n "
308+ elif private_key :
202309 if "-----END " in text and "PRIVATE KEY-----" in text :
203310 private_key = False
204311 continue
205- target .write (self .text (text ).encode ("utf-8" ))
312+ if not include_record :
313+ continue
314+ pending .append (text )
315+ pending_bytes += len (line )
316+ # Keep complete lines together while amortizing regex and ZIP writes
317+ # across a bounded batch instead of repeating them for every line.
318+ if pending_bytes >= 64 * 1024 :
319+ target .write (self .text ("" .join (pending )).encode ("utf-8" ))
320+ pending .clear ()
321+ pending_bytes = 0
322+ if pending :
323+ target .write (self .text ("" .join (pending )).encode ("utf-8" ))
206324 return truncated
207325
208326
209327def collect_diagnostics (
210- config : dict , core : Any = None , * , redact : bool = True
328+ config : dict ,
329+ core : Any = None ,
330+ * ,
331+ redact : bool = True ,
332+ webui_version : str | None = None ,
211333) -> tuple [dict , DiagnosticRedactor ]:
212334 """Snapshot configuration, plugin status and environment for a log export.
213335
214336 Args:
215337 config: Running global configuration.
216338 core: Running lifecycle, when metadata is available.
217339 redact: Whether to mask credentials throughout the bundle.
340+ webui_version: Version declared by the active WebUI assets, if available.
218341
219342 Returns:
220343 JSON sections and a redactor seeded from all available credentials.
@@ -281,6 +404,7 @@ def collect_diagnostics(
281404 pass
282405 sections ["environment.json" ] = {
283406 "astrbot_version" : __version__ ,
407+ "webui_version" : webui_version ,
284408 "commit" : commit ,
285409 "python_version" : platform .python_version (),
286410 "os" : platform .system (),
0 commit comments