|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +# \\ SPIKE: Secure your secrets with SPIFFE. — https://spike.ist/ |
| 4 | +# \\\\\ Copyright 2024-present SPIKE contributors. |
| 5 | +# \\\\\\\ SPDX-License-Identifier: Apache-2.0 |
| 6 | + |
| 7 | +# Live recovery/restore drill for the bare-metal dev environment. |
| 8 | +# |
| 9 | +# Run this from the repository root, in a second terminal, after |
| 10 | +# `make start` has completed cleanly (SPIRE server and agent, three |
| 11 | +# Keepers, and Nexus with a persistent backend are all running). Run it |
| 12 | +# from the same shell environment as `make start` so a restarted Nexus |
| 13 | +# inherits the same configuration. |
| 14 | +# |
| 15 | +# The drill proves the break-the-glass runbook end to end |
| 16 | +# (see https://spike.ist/operations/recovery/): |
| 17 | +# |
| 18 | +# 1. Writes a marker secret while the system is healthy. |
| 19 | +# 2. Exports recovery shards via `spike operator recover`. |
| 20 | +# 3. Simulates a crash: kills Nexus and all Keepers (the Keepers lose |
| 21 | +# their in-memory shards, so auto-recovery is impossible). |
| 22 | +# 4. Restarts Nexus alone and feeds the shards back one by one via |
| 23 | +# `spike operator restore`, using its non-interactive stdin mode. |
| 24 | +# 5. Verifies the marker secret reads back after the restore. |
| 25 | +# |
| 26 | +# The Pilot's SPIRE entry is rotated through the recover and restore |
| 27 | +# roles along the way; a trap reverts it to the superuser role on every |
| 28 | +# exit path. The Keepers stay down when the drill ends: run `make kill` |
| 29 | +# and `make start` afterward to return to a pristine environment. |
| 30 | + |
| 31 | +set -u |
| 32 | + |
| 33 | +DRILL_MARKER_PATH="drill/recovery-marker" |
| 34 | +DRILL_MARKER_VALUE="drill-$(date +%s)-$$" |
| 35 | +RECOVERY_DIR="${SPIKE_PILOT_RECOVERY_DIR:-$HOME/.spike/recover}" |
| 36 | +NEXUS_LOG="$(mktemp -t spike-drill-nexus.XXXXXX.log)" |
| 37 | + |
| 38 | +say() { |
| 39 | + echo "" |
| 40 | + echo "drill: $*" |
| 41 | +} |
| 42 | + |
| 43 | +fail() { |
| 44 | + echo "" >&2 |
| 45 | + echo "drill: FAIL: $*" >&2 |
| 46 | + exit 1 |
| 47 | +} |
| 48 | + |
| 49 | +# Returns the Pilot's current role by asking the SPIRE server which of |
| 50 | +# the known role SPIFFE IDs has a registration entry. |
| 51 | +current_pilot_role() { |
| 52 | + for role in superuser recover restore; do |
| 53 | + if spire-server entry show \ |
| 54 | + -spiffeID "spiffe://spike.ist/spike/pilot/role/$role" 2>/dev/null | |
| 55 | + grep -q "Entry ID"; then |
| 56 | + echo "$role" |
| 57 | + return 0 |
| 58 | + fi |
| 59 | + done |
| 60 | + echo "unknown" |
| 61 | +} |
| 62 | + |
| 63 | +# Reverts the Pilot's entry to the superuser role no matter which role |
| 64 | +# the drill died in. Runs on every exit path via the trap below. |
| 65 | +ensure_superuser_role() { |
| 66 | + case "$(current_pilot_role)" in |
| 67 | + recover) |
| 68 | + say "reverting the Pilot entry from recover to superuser..." |
| 69 | + ./hack/bare-metal/entry/spire-server-entry-recover-revert.sh |
| 70 | + ;; |
| 71 | + restore) |
| 72 | + say "reverting the Pilot entry from restore to superuser..." |
| 73 | + ./hack/bare-metal/entry/spire-server-entry-restore-revert.sh |
| 74 | + ;; |
| 75 | + superuser) |
| 76 | + : |
| 77 | + ;; |
| 78 | + *) |
| 79 | + echo "drill: WARNING: could not determine the Pilot role;" >&2 |
| 80 | + echo "drill: restore it manually with" >&2 |
| 81 | + echo "drill: ./hack/bare-metal/entry/spire-server-entry-su-register.sh" >&2 |
| 82 | + ;; |
| 83 | + esac |
| 84 | +} |
| 85 | + |
| 86 | +trap ensure_superuser_role EXIT |
| 87 | + |
| 88 | +# Retries a command until it succeeds or the attempts run out. |
| 89 | +retry() { |
| 90 | + local attempts="$1" |
| 91 | + local delay="$2" |
| 92 | + shift 2 |
| 93 | + |
| 94 | + local i |
| 95 | + for ((i = 1; i <= attempts; i++)); do |
| 96 | + if "$@"; then |
| 97 | + return 0 |
| 98 | + fi |
| 99 | + sleep "$delay" |
| 100 | + done |
| 101 | + return 1 |
| 102 | +} |
| 103 | + |
| 104 | +# --- Preflight ------------------------------------------------------------ |
| 105 | + |
| 106 | +[ -x ./bin/spike ] || fail "run this from the repository root" \ |
| 107 | + "(./bin/spike not found; did make start build the binaries?)" |
| 108 | + |
| 109 | +for b in spike spire-server; do |
| 110 | + command -v "$b" >/dev/null 2>&1 || fail "'$b' is not on PATH" |
| 111 | +done |
| 112 | + |
| 113 | +command -v spike | xargs test "$(pwd)/bin/spike" -ef || |
| 114 | + fail "PATH resolves 'spike' outside $(pwd)/bin" |
| 115 | + |
| 116 | +pgrep -x nexus >/dev/null || fail "Nexus is not running (run make start)" |
| 117 | +pgrep -x keeper >/dev/null || fail "no Keepers running (run make start)" |
| 118 | +pgrep -x spire-server >/dev/null || fail "SPIRE server is not running" |
| 119 | + |
| 120 | +if [ "${SPIKE_NEXUS_BACKEND_STORE:-}" = "memory" ]; then |
| 121 | + fail "the drill needs a persistent backend;" \ |
| 122 | + "unset SPIKE_NEXUS_BACKEND_STORE" |
| 123 | +fi |
| 124 | + |
| 125 | +if [ "$(current_pilot_role)" != "superuser" ]; then |
| 126 | + fail "the Pilot entry is not in the superuser role;" \ |
| 127 | + "revert it before running the drill" |
| 128 | +fi |
| 129 | + |
| 130 | +# --- 1. Write a marker secret while healthy ------------------------------- |
| 131 | + |
| 132 | +say "writing the marker secret ($DRILL_MARKER_PATH)..." |
| 133 | +spike secret put "$DRILL_MARKER_PATH" value="$DRILL_MARKER_VALUE" || |
| 134 | + fail "could not write the marker secret" |
| 135 | + |
| 136 | +# Note: the Pilot prints its output on stderr (cobra's Print* default), |
| 137 | +# so merge the streams before grepping, like the other harness scripts. |
| 138 | +spike secret get "$DRILL_MARKER_PATH" 2>&1 | |
| 139 | + grep -q "$DRILL_MARKER_VALUE" || |
| 140 | + fail "could not read the marker secret back while healthy" |
| 141 | + |
| 142 | +# --- 2. Export recovery shards --------------------------------------------- |
| 143 | + |
| 144 | +say "rotating the Pilot entry to the recover role..." |
| 145 | +./hack/bare-metal/entry/spire-server-entry-recover-register.sh || |
| 146 | + fail "could not register the recover role" |
| 147 | + |
| 148 | +recover_once() { |
| 149 | + spike operator recover 2>&1 | grep -q "recovery directory" |
| 150 | +} |
| 151 | + |
| 152 | +say "exporting recovery shards (retrying while the SVID rotates)..." |
| 153 | +retry 5 3 recover_once || fail "spike operator recover did not succeed" |
| 154 | + |
| 155 | +shard_count=$(ls "$RECOVERY_DIR"/spike.recovery.*.txt 2>/dev/null | wc -l) |
| 156 | +[ "$shard_count" -ge 2 ] || |
| 157 | + fail "expected at least 2 shard files in $RECOVERY_DIR," \ |
| 158 | + "found $shard_count" |
| 159 | +say "exported $shard_count shards to $RECOVERY_DIR" |
| 160 | + |
| 161 | +say "rotating the Pilot entry back to superuser..." |
| 162 | +./hack/bare-metal/entry/spire-server-entry-recover-revert.sh || |
| 163 | + fail "could not revert the recover role" |
| 164 | + |
| 165 | +# --- 3. Simulate the crash -------------------------------------------------- |
| 166 | + |
| 167 | +say "simulating a crash: killing Nexus and all Keepers..." |
| 168 | +pkill -x nexus |
| 169 | +pkill -x keeper |
| 170 | + |
| 171 | +crash_complete() { |
| 172 | + ! pgrep -x nexus >/dev/null && ! pgrep -x keeper >/dev/null |
| 173 | +} |
| 174 | +retry 10 1 crash_complete || fail "Nexus/Keeper processes did not exit" |
| 175 | +say "Nexus and Keepers are down; Keeper shards are lost" |
| 176 | + |
| 177 | +# --- 4. Restart Nexus alone and restore ------------------------------------ |
| 178 | + |
| 179 | +say "restarting Nexus alone (log: $NEXUS_LOG)..." |
| 180 | +nohup ./hack/bare-metal/startup/start-nexus.sh \ |
| 181 | + >"$NEXUS_LOG" 2>&1 & |
| 182 | + |
| 183 | +say "rotating the Pilot entry to the restore role..." |
| 184 | +./hack/bare-metal/entry/spire-server-entry-restore-register.sh || |
| 185 | + fail "could not register the restore role" |
| 186 | + |
| 187 | +restored=0 |
| 188 | +for shard_file in "$RECOVERY_DIR"/spike.recovery.*.txt; do |
| 189 | + say "feeding $(basename "$shard_file")..." |
| 190 | + |
| 191 | + feed_shard() { |
| 192 | + output=$(spike operator restore <"$shard_file" 2>&1) |
| 193 | + # Two conditions are transient and worth retrying: Nexus is still |
| 194 | + # starting (comms errors), and the SPIRE agent has not yet rotated |
| 195 | + # the Pilot SVID to the restore role (access_unauthorized). |
| 196 | + if echo "$output" | grep -q "Failed to communicate"; then |
| 197 | + return 1 |
| 198 | + fi |
| 199 | + if echo "$output" | grep -q "access_unauthorized"; then |
| 200 | + return 1 |
| 201 | + fi |
| 202 | + return 0 |
| 203 | + } |
| 204 | + |
| 205 | + retry 20 2 feed_shard || |
| 206 | + fail "could not feed $(basename "$shard_file"):" \ |
| 207 | + "Nexus unreachable or the restore SVID never arrived" |
| 208 | + |
| 209 | + echo "$output" |
| 210 | + if echo "$output" | grep -q "restored and ready"; then |
| 211 | + restored=1 |
| 212 | + break |
| 213 | + fi |
| 214 | +done |
| 215 | + |
| 216 | +[ "$restored" -eq 1 ] || |
| 217 | + fail "fed all shards but Nexus did not report itself restored" |
| 218 | + |
| 219 | +say "rotating the Pilot entry back to superuser..." |
| 220 | +./hack/bare-metal/entry/spire-server-entry-restore-revert.sh || |
| 221 | + fail "could not revert the restore role" |
| 222 | + |
| 223 | +# --- 5. Verify the pre-crash secret ---------------------------------------- |
| 224 | + |
| 225 | +verify_marker() { |
| 226 | + spike secret get "$DRILL_MARKER_PATH" 2>&1 | |
| 227 | + grep -q "$DRILL_MARKER_VALUE" |
| 228 | +} |
| 229 | + |
| 230 | +say "verifying the pre-crash marker secret..." |
| 231 | +retry 10 2 verify_marker || |
| 232 | + fail "the marker secret did not read back after the restore" |
| 233 | + |
| 234 | +# --- Cleanup ---------------------------------------------------------------- |
| 235 | + |
| 236 | +say "cleaning up: deleting the marker secret and the shard files..." |
| 237 | +spike secret delete "$DRILL_MARKER_PATH" >/dev/null 2>&1 |
| 238 | +rm -f "$RECOVERY_DIR"/spike.recovery.*.txt |
| 239 | + |
| 240 | +say "PASS: recovery/restore drill completed successfully." |
| 241 | +echo "" |
| 242 | +echo " A secret written before the crash survived the loss of Nexus" |
| 243 | +echo " and every Keeper, and was readable after a shard-based restore." |
| 244 | +echo "" |
| 245 | +echo " Note: the Keepers are still down. Run 'make kill' and then" |
| 246 | +echo " 'make start' to return to a pristine environment." |
| 247 | +echo "" |
0 commit comments