Skip to content

Commit fca661d

Browse files
feat(runner): collect k3s diagnostics after a failed run
The tmachine conformance playbook saved a bounded set of k3s diagnostics when a k3s lane failed: the k3s service state and journal, routing rules, version, API readiness, nodes, the Agent Sandbox controller's deployment, Pods, leases, events and logs, Sandbox reconciliation status, the gateway's Pods, Services, EndpointSlices, events and current and previous logs, the test client's gateway registrations and status, and memory and disk use. The runner only saved the systemd gateway journal, which is empty on k3s, so a failed k3s or HA k3s row lost that evidence. After a failed run, when the guest has /usr/local/bin/k3s, run the old script (collect-k3s.sh, carried over with only its output handling changed: it prints to stdout and the runner writes diagnostics/k3s.txt). The commands, timeouts, size caps and redaction are unchanged. deb and rpm guests keep the gateway journal only. A failure to collect is reported and never changes the test result. Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
1 parent 4e6a9c4 commit fca661d

2 files changed

Lines changed: 159 additions & 1 deletion

File tree

‎tests/runner/src/collect-k3s.sh‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/usr/bin/env bash
2+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
# Prints the diagnostics for a failed run on a k3s guest to stdout; the runner
6+
# saves them. Carried over from the tmachine conformance playbook's diagnostics
7+
# role. No kubeconfig, Secrets, environment dumps, or complete Pod specifications.
8+
set -u
9+
umask 077
10+
11+
collect() {
12+
printf '\n### %s\n' "$1"
13+
shift
14+
# Bound both stalled commands and unusually large log streams.
15+
timeout 10s "$@" 2>&1 | tail -c 262144
16+
printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
17+
}
18+
19+
collect_log() {
20+
printf '\n### %s\n' "$1"
21+
shift
22+
# Retain startup and recent evidence when later scenarios fill the log.
23+
timeout 10s "$@" 2>&1 | {
24+
head -c 131072
25+
printf '\n### Remaining log tail\n'
26+
tail -c 131072
27+
}
28+
printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
29+
}
30+
31+
{
32+
date --utc --iso-8601=seconds
33+
collect 'K3s service state' systemctl show k3s.service \
34+
-p ActiveState -p SubState -p Result -p NRestarts -p ExecMainStatus \
35+
-p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \
36+
-p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec
37+
collect 'K3s boot journal' journalctl -b --no-pager --lines=500 -u k3s.service
38+
# kube-proxy may program either iptables backend; dump whichever exist.
39+
# shellcheck disable=SC2016 # Expanded by the inner shell.
40+
collect 'Gateway Service routing rules' bash -c '
41+
for save in iptables-nft-save iptables-legacy-save; do
42+
command -v "$save" >/dev/null || continue
43+
echo "# $save"
44+
"$save" 2>/dev/null | awk "/openshell\/openshell:grpc/ { print }"
45+
done'
46+
collect 'K3s version' /usr/local/bin/k3s --version
47+
collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz
48+
collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide
49+
collect 'Agent Sandbox controller deployment' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get deployment agent-sandbox-controller \
50+
-o 'custom-columns=NAME:.metadata.name,GENERATION:.metadata.generation,STATUS:.status'
51+
collect 'Agent Sandbox controller Pods' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \
52+
-o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
53+
collect 'Agent Sandbox leader leases' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get leases \
54+
-o 'custom-columns=NAME:.metadata.name,HOLDER:.spec.holderIdentity,ACQUIRED:.spec.acquireTime,RENEWED:.spec.renewTime,DURATION:.spec.leaseDurationSeconds,TRANSITIONS:.spec.leaseTransitions'
55+
collect 'Agent Sandbox controller events' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get events --sort-by=.lastTimestamp
56+
collect 'Sandbox reconciliation status' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get sandboxes.agents.x-k8s.io \
57+
-o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,CREATED:.metadata.creationTimestamp,GENERATION:.metadata.generation,STATUS:.status'
58+
mapfile -t controller_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \
59+
--selector app=agent-sandbox-controller --output name 2>/dev/null)
60+
for pod in "${controller_pods[@]}"; do
61+
collect_log "Agent Sandbox current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --since=30m --timestamps
62+
collect_log "Agent Sandbox previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --previous --since=30m --timestamps
63+
done
64+
collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
65+
-o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses'
66+
collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \
67+
-o 'custom-columns=NAME:.metadata.name,TYPE:.spec.type,CLUSTERIP:.spec.clusterIP,SELECTOR:.spec.selector,PORTS:.spec.ports'
68+
collect 'Test client gateway registrations' runuser -u tmachine -- openshell gateway list --output json
69+
collect 'Test client gateway status' runuser -u tmachine -- openshell status --output json
70+
collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \
71+
-o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions'
72+
collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp
73+
# Gateway Pods are named by a StatefulSet or a Deployment, depending on the installer.
74+
mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
75+
--selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null)
76+
for pod in "${gateway_pods[@]}"; do
77+
collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps
78+
collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps
79+
done
80+
collect 'Memory' free -m
81+
collect 'Disk' df -h / /var/lib/rancher/k3s
82+
} | sed -E \
83+
-e 's/(Bearer )[A-Za-z0-9._~+\/-]+/\1[REDACTED]/gI' \
84+
-e 's/((token|password|secret|authorization)[" ]*[=:][" ]*)[^ ,"]+/\1[REDACTED]/gI'

‎tests/runner/src/execute.rs‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,9 +346,28 @@ fn print_log_tail(log: &Path) {
346346
}
347347
}
348348

349-
/// Write the OpenShell gateway's journal to `dir/gateway.log`.
349+
/// The diagnostics to print from a k3s guest: the script the tmachine
350+
/// conformance playbook ran, with its output on stdout.
351+
const COLLECT_K3S: &str = include_str!("collect-k3s.sh");
352+
/// Where the k3s installers put the k3s binary; its presence marks a k3s guest.
353+
const K3S_BINARY: &str = "/usr/local/bin/k3s";
354+
/// The script bounds each command itself; this catches one that stalls.
355+
const COLLECT_K3S_TIMEOUT: Duration = Duration::from_mins(3);
356+
357+
/// Write the OpenShell gateway's journal to `dir/gateway.log`, and on a k3s
358+
/// guest the k3s diagnostics to `dir/k3s.txt`. Each is collected even if the
359+
/// other fails; a k3s failure is reported, and a journal failure returned.
350360
async fn collect_diagnostics(session: &Session, dir: &Path) -> Result<()> {
351361
std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
362+
let journal = collect_gateway_journal(session, dir).await;
363+
if let Err(error) = collect_k3s_diagnostics(session, dir).await {
364+
eprintln!("==> collecting k3s diagnostics failed: {error:#}");
365+
}
366+
journal
367+
}
368+
369+
/// Write the OpenShell gateway's journal to `dir/gateway.log`.
370+
async fn collect_gateway_journal(session: &Session, dir: &Path) -> Result<()> {
352371
let output = session
353372
.exec(&[
354373
"sudo",
@@ -368,6 +387,24 @@ async fn collect_diagnostics(session: &Session, dir: &Path) -> Result<()> {
368387
std::fs::write(&log, contents).with_context(|| format!("write {}", log.display()))
369388
}
370389

390+
/// Write the k3s diagnostics to `dir/k3s.txt` when the guest runs k3s.
391+
async fn collect_k3s_diagnostics(session: &Session, dir: &Path) -> Result<()> {
392+
let probe = session.exec(&["test", "-x", K3S_BINARY]).await?;
393+
if !probe.status.success() {
394+
return Ok(());
395+
}
396+
let output = session
397+
.exec_timeout(
398+
&["sudo", "-n", "bash", "-c", COLLECT_K3S],
399+
COLLECT_K3S_TIMEOUT,
400+
)
401+
.await?;
402+
let file = dir.join("k3s.txt");
403+
let mut contents = output.stdout;
404+
contents.extend_from_slice(&output.stderr);
405+
std::fs::write(&file, contents).with_context(|| format!("write {}", file.display()))
406+
}
407+
371408
fn checked(step: &str, output: Output) -> Result<String> {
372409
ensure!(
373410
output.status.success(),
@@ -396,6 +433,43 @@ mod tests {
396433
assert!(matches!(status(1), Status::Error(_)));
397434
}
398435

436+
#[test]
437+
fn k3s_diagnostics_keep_everything_the_old_playbook_collected() {
438+
for wanted in [
439+
"systemctl show k3s.service",
440+
"journalctl -b --no-pager --lines=500 -u k3s.service",
441+
"iptables-nft-save",
442+
"--version",
443+
"get --raw=/readyz",
444+
"get nodes -o wide",
445+
"get deployment agent-sandbox-controller",
446+
"-n agent-sandbox-system get pods",
447+
"-n agent-sandbox-system get leases",
448+
"-n agent-sandbox-system get events",
449+
"get sandboxes.agents.x-k8s.io",
450+
"-n agent-sandbox-system logs",
451+
"-n openshell get pods",
452+
"-n openshell get services",
453+
"openshell gateway list --output json",
454+
"openshell status --output json",
455+
"-n openshell get endpointslices",
456+
"-n openshell get events",
457+
"-n openshell logs",
458+
"--previous",
459+
"free -m",
460+
"df -h",
461+
"[REDACTED]",
462+
] {
463+
assert!(COLLECT_K3S.contains(wanted), "missing {wanted:?}");
464+
}
465+
for unwanted in ["get secret", "--kubeconfig", "-o yaml", "-o json"] {
466+
assert!(
467+
!COLLECT_K3S.contains(unwanted),
468+
"must not collect {unwanted:?}"
469+
);
470+
}
471+
}
472+
399473
#[test]
400474
fn exit_code_separates_test_failures_from_errors() {
401475
assert_eq!(Status::Passed.exit_code(), ExitCode::SUCCESS);

0 commit comments

Comments
 (0)