|
| 1 | +#!/bin/bash |
| 2 | +# Process PostgreSQL/OrioleDB core dumps captured by systemd-coredump into a |
| 3 | +# text summary. |
| 4 | +# |
| 5 | +# Key ideas: |
| 6 | +# - Only cores of the postgres binary are processed; any other core left |
| 7 | +# untouched |
| 8 | +# - No environment variables are ever collected. The GDB extraction (see |
| 9 | +# cmds.gdb, matching OrioleDB's own CI debugging script) does run |
| 10 | +# `thread apply all bt full`, which prints local variable values and can |
| 11 | +# surface fragments of in-memory data (buffer/tuple pointers etc.) - this |
| 12 | +# is a deliberate, reviewed trade-off in favor of debuggability, not an |
| 13 | +# oversight. |
| 14 | +# - Cores are deleted after a successful run or quarantined (metadata only) |
| 15 | +# after MAX_ATTEMPTS failures. |
| 16 | + |
| 17 | +set -euo pipefail |
| 18 | + |
| 19 | +STATE_DIR=/var/lib/orioledb-coredumps/state |
| 20 | +OUTPUT_DIR=/var/lib/orioledb-coredumps/diagnostics |
| 21 | +QUARANTINE_DIR=/var/lib/orioledb-coredumps/quarantine |
| 22 | + |
| 23 | +MAX_ATTEMPTS=3 |
| 24 | +EXTRACTION_TIMEOUT=120 |
| 25 | +MAX_AGE_DAYS=7 |
| 26 | +MAX_TOTAL_BYTES=$((300 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse |
| 27 | + |
| 28 | +GDB_DEBUG_DIR=/var/lib/postgresql/.nix-profile/lib/debug |
| 29 | +GDB_CMDS_FILE=/usr/local/sbin/orioledb-coredump-cmds.gdb |
| 30 | +PGDATA_CURRENT_LOGFILES=/var/lib/postgresql/data/current_logfiles |
| 31 | + |
| 32 | +log() { |
| 33 | + echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" |
| 34 | +} |
| 35 | + |
| 36 | +# Only ever run under orioledb-coredump.service, so this always executes with |
| 37 | +# the unit's hardening (privs, sandboxing) rather than whatever an interactive |
| 38 | +# shell happens to have. |
| 39 | +if [[ -z ${INVOCATION_ID:-} ]]; then |
| 40 | + echo "This script must be run through orioledb-coredump.service" >&2 |
| 41 | + exit 1 |
| 42 | +fi |
| 43 | + |
| 44 | +# orioledb.so has no fixed, predictable path either - there is no |
| 45 | +# /usr/lib/postgresql/lib mirror of it. It lives in the same nix store |
| 46 | +# derivation as the resolved postgres executable, just under lib/ instead |
| 47 | +# of bin/, e.g. .../postgresql-and-plugins-17_20/{bin/.postgres-wrapped, |
| 48 | +# lib/orioledb.so} - so derive it from $exe rather than guessing a path. |
| 49 | +orioledb_lib_path() { |
| 50 | + local exe_dir |
| 51 | + exe_dir=$(dirname "$(dirname "$1")") |
| 52 | + printf '%s/lib/orioledb.so' "$exe_dir" |
| 53 | +} |
| 54 | + |
| 55 | +# coredumpctl on this systemd version (255.4) has no "rm"/"delete" verb - the |
| 56 | +# only reliable way to remove a core is to delete its on-disk Filename path |
| 57 | +# directly. Returns success only if the path is actually gone afterward. |
| 58 | +delete_core() { |
| 59 | + local path="$1" |
| 60 | + [ -z "$path" ] && return 1 |
| 61 | + rm -f -- "$path" |
| 62 | + [ ! -e "$path" ] |
| 63 | +} |
| 64 | + |
| 65 | +# The active postgresql log file has an unpredictable name and can be |
| 66 | +# csvlog, stderr-text, or both depending on config (this AMI defaults to |
| 67 | +# csvlog-only, e.g. /var/log/postgresql/postgresql.csv - the stderr-format |
| 68 | +# postgresql.log stops receiving anything the moment the logging collector |
| 69 | +# switches over at startup). PGDATA/current_logfiles is postgres's own, |
| 70 | +# always-current record of the real path(s); prefer csvlog, fall back to |
| 71 | +# stderr. Either format still starts each line with a literal timestamp, so |
| 72 | +# the grep -F substring match below works unchanged either way. |
| 73 | +current_postgres_log() { |
| 74 | + [ -f "$PGDATA_CURRENT_LOGFILES" ] || return 0 |
| 75 | + awk '$1 == "csvlog" {p = $2} $1 == "stderr" && !p {p = $2} END {print p}' "$PGDATA_CURRENT_LOGFILES" |
| 76 | +} |
| 77 | + |
| 78 | +enforce_retention() { |
| 79 | + find "$OUTPUT_DIR" -maxdepth 1 -type f -mtime "+${MAX_AGE_DAYS}" -delete 2>/dev/null || true |
| 80 | + |
| 81 | + while true; do |
| 82 | + total=$(du -sb "$OUTPUT_DIR" 2>/dev/null | cut -f1) |
| 83 | + [ -z "$total" ] && break |
| 84 | + [ "$total" -le "$MAX_TOTAL_BYTES" ] && break |
| 85 | + oldest=$(find "$OUTPUT_DIR" -maxdepth 1 -type f -printf '%T@ %p\n' 2>/dev/null | sort -n | head -1 | cut -d' ' -f2-) |
| 86 | + [ -z "$oldest" ] && break |
| 87 | + log "retention: removing oldest bundle $oldest to stay under ${MAX_TOTAL_BYTES} bytes" |
| 88 | + rm -f "$oldest" |
| 89 | + done |
| 90 | +} |
| 91 | + |
| 92 | +# Handles one candidate crash, given its PID: decide whether it's ours to |
| 93 | +# process, extract a diagnostic bundle via GDB, then delete the raw core. |
| 94 | +# (1) look up the crash via coredumpctl |
| 95 | +# (2) decide keep/ignore/quarantine based on prior attempts |
| 96 | +# (3) export the core and run GDB against it |
| 97 | +# (4) write the bundle and delete the raw core. Returns 1 only for failures |
| 98 | +# worth retrying next run (main() logs those); every other outcome is |
| 99 | +# ignored, quarantined, or successfully processed - returns 0. |
| 100 | +process_one() { |
| 101 | + local pid="$1" |
| 102 | + |
| 103 | + # `coredumpctl --json=short info` silently IGNORES --json on this |
| 104 | + # systemd version (255.4) and falls back to the old plain-text format - |
| 105 | + # confirmed against a real build, not just docs. jq then fails to parse |
| 106 | + # it as JSON at all, so every field below would come out empty (wrong |
| 107 | + # boot_id, no storage path, core never gets deleted) with no visible |
| 108 | + # error anywhere except this script's own log. `coredumpctl --json=short |
| 109 | + # list` (used in main()) is unaffected - only the info verb's --json |
| 110 | + # support is broken on this version. Query the journal directly instead |
| 111 | + # - COREDUMP_* fields are systemd-coredump's own structured record of |
| 112 | + # the crash, independent of coredumpctl's info-rendering bug, and |
| 113 | + # journalctl's -o json has no version-specific gap like this. Only the |
| 114 | + # fields actually used below are requested - COREDUMP_ENVIRON in |
| 115 | + # particular is deliberately left out, matching "no environment |
| 116 | + # variables are ever collected" above. |
| 117 | + local journal_json |
| 118 | + journal_json=$(journalctl -o json \ |
| 119 | + MESSAGE_ID=fc2e22bc6ee647b6b90729ab34a250b1 \ |
| 120 | + "COREDUMP_PID=${pid}" \ |
| 121 | + --output-fields=COREDUMP_EXE,COREDUMP_SIGNAL,COREDUMP_SIGNAL_NAME,COREDUMP_TIMESTAMP,COREDUMP_FILENAME,_BOOT_ID \ |
| 122 | + 2>/dev/null | tail -n 1) |
| 123 | + if [ -z "$journal_json" ]; then |
| 124 | + log "pid ${pid}: no matching journal entry for this coredump" |
| 125 | + return 1 |
| 126 | + fi |
| 127 | + |
| 128 | + # join() (not @tsv) because @tsv double-escapes backslashes, which would |
| 129 | + # corrupt systemd-coredump filenames (e.g. "\x2e") and prevent core |
| 130 | + # deletion - none of these fields can contain a literal tab/newline. |
| 131 | + local exe boot_id signal signal_name timestamp_usec filename |
| 132 | + IFS=$'\t' read -r exe boot_id signal signal_name timestamp_usec filename < <( |
| 133 | + jq -r '[.COREDUMP_EXE, ._BOOT_ID, .COREDUMP_SIGNAL, .COREDUMP_SIGNAL_NAME, .COREDUMP_TIMESTAMP, (.COREDUMP_FILENAME // "")] | join("\t")' <<<"$journal_json" |
| 134 | + ) |
| 135 | + # Unlike coredumpctl info's derived "Storage: present/missing" status, |
| 136 | + # COREDUMP_FILENAME is just the path recorded at capture time, with no |
| 137 | + # guarantee it's still on disk - delete_core already treats "already |
| 138 | + # gone" as success (rm -f + existence check), so no separate presence |
| 139 | + # check is needed here. |
| 140 | + local storage_path="$filename" |
| 141 | + # PID alone isn't a safe dedup key long-term (PIDs get reused across |
| 142 | + # boots), so pair it with boot ID - matches "state keyed by boot ID plus |
| 143 | + # dump identifier" from the original design. |
| 144 | + local key="${boot_id}-${pid}" |
| 145 | + local state_file="${STATE_DIR}/${key}" |
| 146 | + |
| 147 | + # .done means "final decision made, never look at this dump again" |
| 148 | + # (processed successfully or quarantined - non-postgres cores are never |
| 149 | + # enumerated in the first place, see main()'s list filter). |
| 150 | + # .attempts only counts *failed* tries, to cap retries before quarantine. |
| 151 | + [ -f "${state_file}.done" ] && return 0 |
| 152 | + |
| 153 | + local attempts=0 |
| 154 | + [ -f "${state_file}.attempts" ] && attempts=$(cat "${state_file}.attempts") |
| 155 | + if [ "$attempts" -ge "$MAX_ATTEMPTS" ]; then |
| 156 | + log "pid ${pid}: exceeded ${MAX_ATTEMPTS} attempts, discarding core (metadata kept in ${QUARANTINE_DIR})" |
| 157 | + jq . <<<"$journal_json" >"${QUARANTINE_DIR}/${key}.info" |
| 158 | + if delete_core "$storage_path"; then |
| 159 | + log "pid ${pid}: raw core deleted" |
| 160 | + else |
| 161 | + log "pid ${pid}: WARNING - could not delete raw core at '${storage_path}'" |
| 162 | + fi |
| 163 | + touch "${state_file}.done" |
| 164 | + return 0 |
| 165 | + fi |
| 166 | + |
| 167 | + # From here on we're committed to actually processing this dump, so |
| 168 | + # count it as an attempt before doing any of the risky (slow, can fail) |
| 169 | + # work below - a crash/timeout past this point still gets retried, up |
| 170 | + # to MAX_ATTEMPTS |
| 171 | + echo $((attempts + 1)) >"${state_file}.attempts" |
| 172 | + |
| 173 | + # coredumpctl stores the core compressed; pull a private, working copy |
| 174 | + # out into a root-only scratch dir before handing it to GDB. The trap |
| 175 | + # guarantees that scratch dir is removed when this function returns, no |
| 176 | + # matter which of the several `return`s below fires. `trap ... RETURN` |
| 177 | + # is NOT scoped to this function - it fires on the next return from ANY |
| 178 | + # function (confirmed: without the self-clearing `trap - RETURN` here, |
| 179 | + # main()'s own `return "$rc"` re-fires this same trap, referencing a |
| 180 | + # $tmpdir that's gone out of scope, which is an unbound-variable crash |
| 181 | + # under `set -u` - so it must clear itself once it's run. |
| 182 | + local tmpdir |
| 183 | + tmpdir=$(mktemp -d /tmp/coredump-XXXXXX) |
| 184 | + chmod 700 "$tmpdir" |
| 185 | + trap 'rm -rf "$tmpdir"; trap - RETURN' RETURN |
| 186 | + |
| 187 | + if ! timeout "$EXTRACTION_TIMEOUT" coredumpctl dump "$pid" --output "${tmpdir}/core" >/dev/null 2>&1; then |
| 188 | + log "pid ${pid}: export failed or timed out" |
| 189 | + return 1 |
| 190 | + fi |
| 191 | + |
| 192 | + # Everything from here to the closing "}" is the diagnostic bundle |
| 193 | + # itself, one section at a time, redirected straight to $bundle - |
| 194 | + # there's no in-memory buffering of it, so a slow/hanging step just |
| 195 | + # shows up as a truncated file rather than blocking the whole write. |
| 196 | + # Timestamp from coredumpctl JSON is microseconds since the epoch, so it |
| 197 | + # converts straight to a UTC date/time with no string parsing - and that |
| 198 | + # same "YYYY-MM-DD HH:MM:SS" form is what postgres's own log line prefix |
| 199 | + # starts with, so it doubles as the grep key below. |
| 200 | + local crash_timestamp |
| 201 | + crash_timestamp=$(date -u -d "@$((timestamp_usec / 1000000))" '+%Y-%m-%d %H:%M:%S') |
| 202 | + |
| 203 | + local bundle="${OUTPUT_DIR}/${key}.txt" |
| 204 | + { |
| 205 | + echo "== OrioleDB/PostgreSQL coredump diagnostic bundle ==" |
| 206 | + echo "generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')" |
| 207 | + echo "pid: ${pid}" |
| 208 | + echo "boot_id: ${boot_id}" |
| 209 | + echo "signal: ${signal} (${signal_name})" |
| 210 | + echo "crash_timestamp: ${crash_timestamp} UTC" |
| 211 | + echo "executable: ${exe}" |
| 212 | + echo |
| 213 | + |
| 214 | + echo "== build ids ==" |
| 215 | + echo "postgres (${exe}):" |
| 216 | + readelf -n "$exe" 2>/dev/null | grep 'Build ID' || echo " (could not read build id)" |
| 217 | + orioledb_lib=$(orioledb_lib_path "$exe") |
| 218 | + if [ -f "$orioledb_lib" ]; then |
| 219 | + echo "orioledb.so (${orioledb_lib}):" |
| 220 | + readelf -n "$orioledb_lib" 2>/dev/null | grep 'Build ID' || echo " (could not read build id)" |
| 221 | + fi |
| 222 | + echo |
| 223 | + |
| 224 | + echo "== gdb backtrace (full), lwlocks, locked pages, argv, shared libraries, registers ==" |
| 225 | + timeout "$EXTRACTION_TIMEOUT" gdb --batch -quiet \ |
| 226 | + -ex "set debug-file-directory ${GDB_DEBUG_DIR}" \ |
| 227 | + -ex "file ${exe}" \ |
| 228 | + -ex "core-file ${tmpdir}/core" \ |
| 229 | + -x "$GDB_CMDS_FILE" \ |
| 230 | + 2>&1 || echo "(gdb extraction failed or timed out)" |
| 231 | + echo |
| 232 | + |
| 233 | + echo "== postgresql.log excerpt around crash ==" |
| 234 | + postgres_log=$(current_postgres_log) |
| 235 | + if [ -n "$postgres_log" ] && [ -f "$postgres_log" ]; then |
| 236 | + grep -F "$crash_timestamp" -A 5 -B 20 "$postgres_log" 2>/dev/null | tail -200 || |
| 237 | + echo "(no log lines found matching ${crash_timestamp} in ${postgres_log})" |
| 238 | + else |
| 239 | + echo "(no crash timestamp or log file available)" |
| 240 | + fi |
| 241 | + } >"$bundle" |
| 242 | + chmod 600 "$bundle" |
| 243 | + |
| 244 | + # Bundle is written either way at this point, even if the raw-core |
| 245 | + # delete below fails - we don't want a delete failure to make us |
| 246 | + # reprocess an already-complete bundle next run. |
| 247 | + touch "${state_file}.done" |
| 248 | + if delete_core "$storage_path"; then |
| 249 | + log "pid ${pid}: wrote ${bundle}, deleted raw core" |
| 250 | + else |
| 251 | + log "pid ${pid}: wrote ${bundle}, but WARNING - could not delete raw core at '${storage_path}'" |
| 252 | + fi |
| 253 | +} |
| 254 | + |
| 255 | +main() { |
| 256 | + mkdir -p "$STATE_DIR" "$OUTPUT_DIR" "$QUARANTINE_DIR" |
| 257 | + chmod 700 "$STATE_DIR" "$OUTPUT_DIR" "$QUARANTINE_DIR" |
| 258 | + |
| 259 | + enforce_retention |
| 260 | + |
| 261 | + # Enumerate via coredumpctl and filter out non-postgres cores |
| 262 | + local rc=0 |
| 263 | + local pid |
| 264 | + while read -r pid; do |
| 265 | + [ -z "$pid" ] && continue |
| 266 | + process_one "$pid" || { |
| 267 | + log "pid ${pid}: processing failed, will retry on next run" |
| 268 | + rc=1 |
| 269 | + } |
| 270 | + done < <(coredumpctl --no-pager --json=short list 2>/dev/null | jq -r '.[] | select(.corefile == "present" and (.exe | endswith("/.postgres-wrapped"))) | .pid') |
| 271 | + |
| 272 | + return "$rc" |
| 273 | +} |
| 274 | + |
| 275 | +main |
0 commit comments