|
| 1 | +#!/usr/bin/env bash |
| 2 | +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 3 | +# SPDX-License-Identifier: Apache-2.0 |
| 4 | + |
| 5 | +# Prints the diagnostics for a failed run on a k3s guest to stdout; the runner |
| 6 | +# saves them. Carried over from the tmachine conformance playbook's diagnostics |
| 7 | +# role. No kubeconfig, Secrets, environment dumps, or complete Pod specifications. |
| 8 | +set -u |
| 9 | +umask 077 |
| 10 | + |
| 11 | +collect() { |
| 12 | + printf '\n### %s\n' "$1" |
| 13 | + shift |
| 14 | + # Bound both stalled commands and unusually large log streams. |
| 15 | + timeout 10s "$@" 2>&1 | tail -c 262144 |
| 16 | + printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" |
| 17 | +} |
| 18 | + |
| 19 | +collect_log() { |
| 20 | + printf '\n### %s\n' "$1" |
| 21 | + shift |
| 22 | + # Retain startup and recent evidence when later scenarios fill the log. |
| 23 | + timeout 10s "$@" 2>&1 | { |
| 24 | + head -c 131072 |
| 25 | + printf '\n### Remaining log tail\n' |
| 26 | + tail -c 131072 |
| 27 | + } |
| 28 | + printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" |
| 29 | +} |
| 30 | + |
| 31 | +{ |
| 32 | + date --utc --iso-8601=seconds |
| 33 | + collect 'K3s service state' systemctl show k3s.service \ |
| 34 | + -p ActiveState -p SubState -p Result -p NRestarts -p ExecMainStatus \ |
| 35 | + -p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \ |
| 36 | + -p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec |
| 37 | + collect 'K3s boot journal' journalctl -b --no-pager --lines=500 -u k3s.service |
| 38 | + # kube-proxy may program either iptables backend; dump whichever exist. |
| 39 | + # shellcheck disable=SC2016 # Expanded by the inner shell. |
| 40 | + collect 'Gateway Service routing rules' bash -c ' |
| 41 | + for save in iptables-nft-save iptables-legacy-save; do |
| 42 | + command -v "$save" >/dev/null || continue |
| 43 | + echo "# $save" |
| 44 | + "$save" 2>/dev/null | awk "/openshell\/openshell:grpc/ { print }" |
| 45 | + done' |
| 46 | + collect 'K3s version' /usr/local/bin/k3s --version |
| 47 | + collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz |
| 48 | + collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide |
| 49 | + collect 'Agent Sandbox controller deployment' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get deployment agent-sandbox-controller \ |
| 50 | + -o 'custom-columns=NAME:.metadata.name,GENERATION:.metadata.generation,STATUS:.status' |
| 51 | + collect 'Agent Sandbox controller Pods' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \ |
| 52 | + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' |
| 53 | + collect 'Agent Sandbox leader leases' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get leases \ |
| 54 | + -o 'custom-columns=NAME:.metadata.name,HOLDER:.spec.holderIdentity,ACQUIRED:.spec.acquireTime,RENEWED:.spec.renewTime,DURATION:.spec.leaseDurationSeconds,TRANSITIONS:.spec.leaseTransitions' |
| 55 | + collect 'Agent Sandbox controller events' /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get events --sort-by=.lastTimestamp |
| 56 | + collect 'Sandbox reconciliation status' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get sandboxes.agents.x-k8s.io \ |
| 57 | + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,CREATED:.metadata.creationTimestamp,GENERATION:.metadata.generation,STATUS:.status' |
| 58 | + mapfile -t controller_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system get pods \ |
| 59 | + --selector app=agent-sandbox-controller --output name 2>/dev/null) |
| 60 | + for pod in "${controller_pods[@]}"; do |
| 61 | + collect_log "Agent Sandbox current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --since=30m --timestamps |
| 62 | + collect_log "Agent Sandbox previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n agent-sandbox-system logs "$pod" --previous --since=30m --timestamps |
| 63 | + done |
| 64 | + collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ |
| 65 | + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' |
| 66 | + collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \ |
| 67 | + -o 'custom-columns=NAME:.metadata.name,TYPE:.spec.type,CLUSTERIP:.spec.clusterIP,SELECTOR:.spec.selector,PORTS:.spec.ports' |
| 68 | + collect 'Test client gateway registrations' runuser -u tmachine -- openshell gateway list --output json |
| 69 | + collect 'Test client gateway status' runuser -u tmachine -- openshell status --output json |
| 70 | + collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \ |
| 71 | + -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions' |
| 72 | + collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp |
| 73 | + # Gateway Pods are named by a StatefulSet or a Deployment, depending on the installer. |
| 74 | + mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ |
| 75 | + --selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null) |
| 76 | + for pod in "${gateway_pods[@]}"; do |
| 77 | + collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps |
| 78 | + collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps |
| 79 | + done |
| 80 | + collect 'Memory' free -m |
| 81 | + collect 'Disk' df -h / /var/lib/rancher/k3s |
| 82 | +} | sed -E \ |
| 83 | + -e 's/(Bearer )[A-Za-z0-9._~+\/-]+/\1[REDACTED]/gI' \ |
| 84 | + -e 's/((token|password|secret|authorization)[" ]*[=:][" ]*)[^ ,"]+/\1[REDACTED]/gI' |
0 commit comments