Skip to content

Commit 5881367

Browse files
authored
fix(dashboard): complete diagnostic export improvements (#10659)
* feat(dashboard): improve diagnostic export and add time ranges * refactor(dashboard): move diagnostics into maintenance settings
1 parent f52fe19 commit 5881367

24 files changed

Lines changed: 785 additions & 64 deletions

File tree

‎astrbot/core/utils/trace_store.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,8 @@ def list_traces(
490490
errors: bool = False,
491491
models: bool = False,
492492
plugins: bool = False,
493+
start_time: int | None = None,
494+
end_time: int | None = None,
493495
) -> dict:
494496
"""Read one numbered or keyset page, never all stored span bodies.
495497
@@ -501,6 +503,8 @@ def list_traces(
501503
errors: Require an error.
502504
models: Require an Agent call.
503505
plugins: Require a plugin call.
506+
start_time: Inclusive minimum trace start time, in Unix seconds.
507+
end_time: Inclusive maximum trace start time, in Unix seconds.
504508
505509
Returns:
506510
Summaries, filtered count, next cursor, and storage health.
@@ -515,6 +519,12 @@ def list_traces(
515519
if page < 1 or (cursor and page != 1):
516520
raise ValueError("Invalid trace page.")
517521
conditions, values = [], []
522+
if start_time is not None:
523+
conditions.append("start_ns >= ?")
524+
values.append(start_time * 1_000_000_000)
525+
if end_time is not None:
526+
conditions.append("start_ns <= ?")
527+
values.append(end_time * 1_000_000_000)
518528
for key, enabled in (
519529
("has_errors", errors),
520530
("has_models", models),

‎astrbot/dashboard/api/app.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,12 @@ def create_dashboard_asgi_app(
126126
files=FileService(),
127127
knowledge_bases=KnowledgeBaseService(core_lifecycle),
128128
live_chat=LiveChatService(db, core_lifecycle),
129-
logs=LogService(log_broker, core_lifecycle.astrbot_config, core=core_lifecycle),
129+
logs=LogService(
130+
log_broker,
131+
core_lifecycle.astrbot_config,
132+
core=core_lifecycle,
133+
dashboard_static_folder=static_folder,
134+
),
130135
bots=BotConfigService(core_lifecycle),
131136
platforms=PlatformService(core_lifecycle),
132137
providers=ProviderConfigService(core_lifecycle),

‎astrbot/dashboard/api/logs.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,11 +87,18 @@ async def live_logs(
8787
@router.get("/logs/export")
8888
async def export_logs(
8989
redact: bool = Query(default=True),
90+
start_time: int | None = Query(default=None, ge=0),
91+
end_time: int | None = Query(default=None, ge=0),
9092
_username: str = Depends(require_dashboard_user),
9193
service: LogService = Depends(get_service),
9294
):
9395
try:
94-
archive_path = await asyncio.to_thread(service.export_logs, redact=redact)
96+
archive_path = await asyncio.to_thread(
97+
service.export_logs,
98+
redact=redact,
99+
start_time=start_time,
100+
end_time=end_time,
101+
)
95102
except LogServiceError as exc:
96103
_raise_log_error(exc)
97104
return FileResponse(

‎astrbot/dashboard/services/diagnostic_service.py‎

Lines changed: 148 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import platform
99
import re
1010
import subprocess
11+
from datetime import datetime
1112
from pathlib import Path
1213
from typing import Any, BinaryIO
1314
from urllib.parse import quote, quote_plus
@@ -17,7 +18,7 @@
1718

1819
REDACTED = "[REDACTED]"
1920
_SECRET_KEY = re.compile(
20-
r"password|passwd|secret|token$|api[_-]?key|access[_-]?key|private[_-]?key|"
21+
r"password|passwd|secret|token$|api[_-]?key|access[_-]?key|private[_-]?key|aes[_-]?key|"
2122
r"authorization|cookie|credential|(?:client|app|signing|encrypt(?:ion)?)[_-]?key|"
2223
r"recovery[_-]?code|(?:^|\.)key$|^headers$|^env$",
2324
re.IGNORECASE,
@@ -26,13 +27,14 @@
2627
_AUTH = re.compile(r"\b(Bearer|Basic|ApiKey)\s+[^\s\"'<>]+", re.IGNORECASE)
2728
_COOKIE = re.compile(r"(\b(?:set-cookie|cookie)\s*:\s*)[^\r\n]+", re.IGNORECASE)
2829
_URL_AUTH = re.compile(r"(https?://)[^/\s@]+@", re.IGNORECASE)
29-
_ASSIGNMENT = re.compile(
30-
r"((?:[\w.-]*(?:token|secret|password|passwd|api[_-]?key|access[_-]?key|"
31-
r"(?:client|app|signing|encrypt(?:ion)?)[_-]?key|recovery[_-]?code)"
32-
r"[\w.-]*|authorization|cookie|\bkey)[\"']?\s*[:=]\s*)"
33-
r"(\"[^\"]*\"|'[^']*'|[^\s&,;}]+)",
30+
_ASSIGNMENT = re.compile(r"(?<![\w.-])([\w.-]+)[\"']?\s*[:=]\s*")
31+
_ASSIGNMENT_KEY = re.compile(
32+
r"token|secret|password|passwd|api[_-]?key|access[_-]?key|aes[_-]?key|"
33+
r"(?:client|app|signing|encrypt(?:ion)?)[_-]?key|recovery[_-]?code|"
34+
r"(?:authorization|cookie|\bkey)$",
3435
re.IGNORECASE,
3536
)
37+
_ASSIGNMENT_VALUE = re.compile(r"\"[^\"]*\"|'[^']*'|[^\s&,;}]+")
3638
_TOKEN = re.compile(
3739
r"\b(?:sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9_]{20,}|"
3840
r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})"
@@ -49,6 +51,10 @@
4951
r")(?!\w|\.\w)"
5052
)
5153
MAX_LOG_LINE_BYTES = 1024 * 1024
54+
_LOG_TIMESTAMP = re.compile(
55+
r"^\[?(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}"
56+
r"(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?)(?:\]|\s|$)"
57+
)
5258

5359

5460
class DiagnosticRedactor:
@@ -57,6 +63,7 @@ class DiagnosticRedactor:
5763
def __init__(self, *, enabled: bool = True) -> None:
5864
self.enabled = enabled
5965
self.secrets: set[str] = set()
66+
self._sorted_secrets: list[str] | None = None
6067
self.public_ips: dict[str, str] = {}
6168

6269
def collect(self, value: Any, sensitive: bool = False) -> None:
@@ -79,6 +86,7 @@ def collect(self, value: Any, sensitive: bool = False) -> None:
7986
self.secrets.update(
8087
(text, quote(text, safe=""), quote_plus(text), json.dumps(text)[1:-1])
8188
)
89+
self._sorted_secrets = None
8290

8391
def redact(self, value: Any, sensitive: bool = False) -> Any:
8492
"""Return a sanitized copy while retaining containers and boolean flags.
@@ -118,15 +126,59 @@ def text(self, value: str) -> str:
118126
"""
119127
if not self.enabled:
120128
return value
121-
value = _ANSI.sub("", value)
122-
value = _PRIVATE_KEY.sub(REDACTED, value)
123-
for secret in sorted(self.secrets, key=len, reverse=True):
124-
value = value.replace(secret, REDACTED)
125-
value = _AUTH.sub(lambda match: f"{match[1]} {REDACTED}", value)
126-
value = _COOKIE.sub(lambda match: f"{match[1]}{REDACTED}", value)
127-
value = _URL_AUTH.sub(lambda match: f"{match[1]}{REDACTED}@", value)
128-
value = _ASSIGNMENT.sub(lambda match: f"{match[1]}{REDACTED}", value)
129-
value = _TOKEN.sub(REDACTED, value)
129+
if "\x1b[" in value:
130+
value = _ANSI.sub("", value)
131+
if "PRIVATE KEY-----" in value:
132+
value = _PRIVATE_KEY.sub(REDACTED, value)
133+
if self.secrets:
134+
if self._sorted_secrets is None:
135+
self._sorted_secrets = sorted(self.secrets, key=len, reverse=True)
136+
for secret in self._sorted_secrets:
137+
value = value.replace(secret, REDACTED)
138+
# These cheap substring checks avoid running every regex on ordinary
139+
# ASCII log lines. Unicode case-insensitive regex matches can include
140+
# characters that lower() does not normalize to the ASCII hint.
141+
lowered = value.lower()
142+
ascii_only = value.isascii()
143+
if not ascii_only or any(
144+
prefix in lowered for prefix in ("bearer", "basic", "apikey")
145+
):
146+
value = _AUTH.sub(lambda match: f"{match[1]} {REDACTED}", value)
147+
if not ascii_only or "cookie" in lowered:
148+
value = _COOKIE.sub(lambda match: f"{match[1]}{REDACTED}", value)
149+
if "://" in value and "@" in value:
150+
value = _URL_AUTH.sub(lambda match: f"{match[1]}{REDACTED}@", value)
151+
if not ascii_only or any(
152+
hint in lowered
153+
for hint in (
154+
"token",
155+
"secret",
156+
"password",
157+
"passwd",
158+
"key",
159+
"code",
160+
"authorization",
161+
"cookie",
162+
)
163+
):
164+
# Scan assignment prefixes without consuming ordinary values, which
165+
# may themselves contain a nested credential assignment.
166+
parts = []
167+
end = 0
168+
for match in _ASSIGNMENT.finditer(value):
169+
if match.start() < end or not _ASSIGNMENT_KEY.search(match[1]):
170+
continue
171+
assigned = _ASSIGNMENT_VALUE.match(value, match.end())
172+
if assigned is not None:
173+
parts.extend((value[end : match.end()], REDACTED))
174+
end = assigned.end()
175+
parts.append(value[end:])
176+
value = "".join(parts)
177+
if any(
178+
prefix in value
179+
for prefix in ("sk-", "ghp_", "gho_", "ghu_", "ghs_", "ghr_", "eyJ")
180+
):
181+
value = _TOKEN.sub(REDACTED, value)
130182
parts = []
131183
end = 0
132184
for match in _IP_LITERAL.finditer(value):
@@ -136,6 +188,12 @@ def text(self, value: str) -> str:
136188
literal[1:-1] if literal.startswith("[") else literal.rstrip(".")
137189
)
138190
host, separator, scope = candidate.partition("%")
191+
if (
192+
":" in host
193+
and "::" not in host
194+
and host.count(":") != (6 if "." in host else 7)
195+
):
196+
continue
139197
try:
140198
address = ipaddress.ip_address(host)
141199
except ValueError:
@@ -156,18 +214,29 @@ def text(self, value: str) -> str:
156214
parts.append(value[end:])
157215
return "".join(parts)
158216

159-
def copy_log(self, source: BinaryIO, target: BinaryIO, limit: int) -> bool:
217+
def copy_log(
218+
self,
219+
source: BinaryIO,
220+
target: BinaryIO,
221+
limit: int,
222+
*,
223+
start_time: int | None = None,
224+
end_time: int | None = None,
225+
) -> bool:
160226
"""Stream complete text lines, skipping oversized lines and private keys.
161227
162228
Args:
163229
source: An already verified regular file, positioned at its start.
164230
target: Open ZIP entry for redacted text.
165231
limit: Maximum source bytes to consume, fixed before reading.
232+
start_time: Inclusive minimum record time, in Unix seconds.
233+
end_time: Inclusive maximum record time, in Unix seconds.
166234
167235
Returns:
168236
Whether content was omitted to respect line or file limits.
169237
"""
170-
if not self.enabled:
238+
filtering = start_time is not None or end_time is not None
239+
if not self.enabled and not filtering:
171240
remaining = limit
172241
while remaining:
173242
chunk = source.read(min(remaining, MAX_LOG_LINE_BYTES))
@@ -179,42 +248,96 @@ def copy_log(self, source: BinaryIO, target: BinaryIO, limit: int) -> bool:
179248
truncated = False
180249
oversized = False
181250
private_key = False
251+
pending: list[str] = []
252+
pending_bytes = 0
182253
remaining = limit
254+
include_record = not filtering
183255
while remaining:
184256
line = source.readline(min(remaining, MAX_LOG_LINE_BYTES + 1))
185257
if not line:
186258
break
187259
remaining -= len(line)
260+
if filtering and not oversized and not private_key:
261+
# Timestamp-free continuation lines inherit their record's
262+
# selection; content before the first dated record is excluded.
263+
prefix = line[:160].decode("utf-8", errors="replace")
264+
match = _LOG_TIMESTAMP.match(_ANSI.sub("", prefix))
265+
timestamp = None
266+
dated_record = match is not None
267+
try:
268+
if match is not None:
269+
timestamp = datetime.fromisoformat(
270+
match[1].replace("Z", "+00:00").replace(",", ".")
271+
).timestamp()
272+
elif prefix.startswith("{"):
273+
record = json.loads(line)
274+
if isinstance(record, dict):
275+
dated_record = True
276+
if "time_unix_nano" in record:
277+
timestamp = int(record["time_unix_nano"]) / 1e9
278+
elif isinstance(record.get("time"), (int, float)):
279+
timestamp = record["time"]
280+
elif "start_time_unix_nano" in record:
281+
timestamp = int(record["start_time_unix_nano"]) / 1e9
282+
except (ValueError, TypeError, OverflowError, OSError):
283+
pass
284+
if dated_record:
285+
include_record = (
286+
timestamp is not None
287+
and (start_time is None or timestamp >= start_time)
288+
and (end_time is None or timestamp <= end_time)
289+
)
188290
complete = line.endswith(b"\n") or len(line) <= MAX_LOG_LINE_BYTES
189291
if oversized or not complete:
190292
oversized = not line.endswith(b"\n")
191-
truncated = True
192-
continue
293+
if self.enabled:
294+
truncated = True
295+
continue
193296
# Never emit a fragment cut at the export boundary: it could be a token.
194297
if not remaining and not line.endswith(b"\n") and source.read(1):
195298
truncated = True
196299
break
300+
if not self.enabled:
301+
if include_record:
302+
target.write(line)
303+
continue
197304
text = line.decode("utf-8", errors="replace")
198305
if "-----BEGIN " in text and "PRIVATE KEY-----" in text:
199-
private_key = True
200-
target.write((REDACTED + "\n").encode())
201-
if private_key:
306+
private_key = not ("-----END " in text and "PRIVATE KEY-----" in text)
307+
text = REDACTED + "\n"
308+
elif private_key:
202309
if "-----END " in text and "PRIVATE KEY-----" in text:
203310
private_key = False
204311
continue
205-
target.write(self.text(text).encode("utf-8"))
312+
if not include_record:
313+
continue
314+
pending.append(text)
315+
pending_bytes += len(line)
316+
# Keep complete lines together while amortizing regex and ZIP writes
317+
# across a bounded batch instead of repeating them for every line.
318+
if pending_bytes >= 64 * 1024:
319+
target.write(self.text("".join(pending)).encode("utf-8"))
320+
pending.clear()
321+
pending_bytes = 0
322+
if pending:
323+
target.write(self.text("".join(pending)).encode("utf-8"))
206324
return truncated
207325

208326

209327
def collect_diagnostics(
210-
config: dict, core: Any = None, *, redact: bool = True
328+
config: dict,
329+
core: Any = None,
330+
*,
331+
redact: bool = True,
332+
webui_version: str | None = None,
211333
) -> tuple[dict, DiagnosticRedactor]:
212334
"""Snapshot configuration, plugin status and environment for a log export.
213335
214336
Args:
215337
config: Running global configuration.
216338
core: Running lifecycle, when metadata is available.
217339
redact: Whether to mask credentials throughout the bundle.
340+
webui_version: Version declared by the active WebUI assets, if available.
218341
219342
Returns:
220343
JSON sections and a redactor seeded from all available credentials.
@@ -281,6 +404,7 @@ def collect_diagnostics(
281404
pass
282405
sections["environment.json"] = {
283406
"astrbot_version": __version__,
407+
"webui_version": webui_version,
284408
"commit": commit,
285409
"python_version": platform.python_version(),
286410
"os": platform.system(),

0 commit comments

Comments
 (0)