Skip to content

Commit 8760e5f

Browse files
joecoralljoecorall
committed
[minor] Automate Traefik certificate rotation
Add an idempotent Let's Encrypt hook with certificate validation, Traefik-only recreation, health verification, and Slack failure alerts.
1 parent 4411ab3 commit 8760e5f

5 files changed

Lines changed: 194 additions & 8 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@ ping: ## Ensure site is available.
4040
./scripts/ping.sh
4141

4242
lehigh-certs: ## Place Lehigh's certs into traefik's cert store
43-
./scripts/lehigh-certs.sh
43+
sudo ./scripts/lehigh-certs.sh

‎README.md‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,40 @@ ArchivesSpace's docker compose stack is bound to the docker service so when the
3939

4040
## TLS Certs
4141

42-
Traefik is configured to use Lehigh's wildcard cert. When copying the cert for traefik, ensure the full chain is in `./certs/cert.pem`
42+
Traefik uses Lehigh's Let's Encrypt wildcard certificate through Compose
43+
secrets. The weekly wildcard updater invokes `/usr/local/sbin/local-cert-hook`
44+
after it downloads these files:
45+
46+
- Full certificate chain: `/etc/ssl/certs/le/lib.lehigh.edu.pem`
47+
- Private key: `/etc/ssl/private/le/lib.lehigh.edu.key`
48+
49+
Install the hook and its root-only configuration on each Docker host:
50+
51+
```
52+
cd /opt/archivesspace
53+
sudo install -o root -g root -m 0600 \
54+
scripts/local-cert-hook.env.example /etc/default/local-cert-hook
55+
sudoedit /etc/default/local-cert-hook # set SLACK_WEBHOOK and EXPECTED_HOST
56+
sudo install -o root -g root -m 0755 \
57+
scripts/lehigh-certs.sh /usr/local/sbin/local-cert-hook
58+
sudo chown root:root /opt/archivesspace /opt/archivesspace/docker-compose.yml certs
59+
sudo chmod go-w /opt/archivesspace /opt/archivesspace/docker-compose.yml certs
60+
```
61+
62+
The hook validates the full chain, hostname, expiry, private key, and key/cert
63+
pair before changing anything. It compares SHA-256 checksums with the files
64+
already backing the Compose secrets, so an unchanged renewal exits without
65+
recreating Traefik. When either file has changed, it atomically installs both
66+
and force-recreates only Traefik so Docker remounts the secrets. Any validation,
67+
Compose, or healthcheck failure is sent to Slack and logged to stderr and
68+
syslog.
69+
70+
The hook and every path it uses for root-level Compose operations must remain
71+
root-owned and not group/world-writable. Salt can manage the two installed
72+
files instead of the commands above; keep `/etc/default/local-cert-hook` at
73+
mode `0600` because it contains the Slack webhook.
74+
75+
To invoke the same hook manually:
4376

4477
```
4578
cd /opt/archivesspace

‎docker-compose.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,9 @@ services:
112112
- traefik-data:/data
113113
healthcheck:
114114
test: traefik healthcheck --ping
115+
interval: 2s
116+
timeout: 2s
117+
retries: 15
115118
start_period: 5s
116119
depends_on:
117120
app:

‎scripts/lehigh-certs.sh‎

Lines changed: 144 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,149 @@
11
#!/usr/bin/env bash
22

3-
set -eou pipefail
3+
set -euo pipefail
44

5-
echo "Copying Lehigh's certs into traefik"
5+
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
6+
export PATH
7+
umask 077
68

7-
cat /etc/ssl/certs/lib.lehigh.edu.crt /etc/ssl/certs/gd_bundle-g2-g1.crt | sudo tee certs/cert.pem
9+
ERROR_MESSAGE="unexpected error"
10+
STAGED_CERT=""
11+
STAGED_KEY=""
812

9-
sudo cp /etc/ssl/private/lib.lehigh.edu.key certs/privkey.pem
10-
sudo chmod 700 certs/privkey.pem
11-
sudo chown root certs/privkey.pem
13+
send_slack_message() {
14+
local payload
15+
16+
[[ -n "${SLACK_WEBHOOK:-}" ]] || return 1
17+
payload="$(jq -cn --arg msg "$*" '{msg: $msg}')"
18+
19+
curl --fail --silent --show-error --output /dev/null \
20+
--connect-timeout 5 --max-time 15 \
21+
--header 'Content-Type: application/json' \
22+
--request POST --data-binary "${payload}" "${SLACK_WEBHOOK}"
23+
}
24+
25+
cleanup() {
26+
[[ -z "${STAGED_CERT}" ]] || rm -f -- "${STAGED_CERT}"
27+
[[ -z "${STAGED_KEY}" ]] || rm -f -- "${STAGED_KEY}"
28+
}
29+
30+
handle_error() {
31+
local exit_code="$1"
32+
local host
33+
local message
34+
35+
trap - ERR
36+
set +e
37+
host="$(hostname --fqdn 2>/dev/null || hostname)"
38+
message="🚨 ${ALERT_CONTEXT:-TLS certificate rollout} failed on ${host}: ${ERROR_MESSAGE} 🚨"
39+
40+
printf '%s\n' "${message}" >&2
41+
logger --priority daemon.err --tag local-cert-hook -- "${message}" || true
42+
send_slack_message "${message}" || \
43+
printf '%s\n' 'Slack alert could not be sent; check SLACK_WEBHOOK, curl, and jq.' >&2
44+
45+
exit "${exit_code}"
46+
}
47+
48+
fail() {
49+
ERROR_MESSAGE="$*"
50+
return 1
51+
}
52+
53+
trap 'handle_error "$?"' ERR
54+
trap cleanup EXIT
55+
56+
HOOK_CONFIG="${LOCAL_CERT_HOOK_CONFIG:-/etc/default/local-cert-hook}"
57+
if [[ -r "${HOOK_CONFIG}" ]]; then
58+
# shellcheck disable=SC1090
59+
source "${HOOK_CONFIG}"
60+
fi
61+
62+
SOURCE_CERT="${SOURCE_CERT:-/etc/ssl/certs/le/lib.lehigh.edu.pem}"
63+
SOURCE_KEY="${SOURCE_KEY:-/etc/ssl/private/le/lib.lehigh.edu.key}"
64+
STACK_DIR="${STACK_DIR:-/opt/archivesspace}"
65+
CERT_DIR="${CERT_DIR:-${STACK_DIR}/certs}"
66+
TARGET_CERT="${CERT_DIR}/cert.pem"
67+
TARGET_KEY="${CERT_DIR}/privkey.pem"
68+
RESTART_REQUIRED="${CERT_DIR}/.traefik-restart-required"
69+
EXPECTED_HOST="${EXPECTED_HOST:-archivesspace.lib.lehigh.edu}"
70+
HEALTHCHECK_TIMEOUT="${HEALTHCHECK_TIMEOUT:-60}"
71+
ALERT_CONTEXT="${ALERT_CONTEXT:-ArchivesSpace TLS certificate rollout}"
72+
73+
COMPOSE=(docker compose --project-directory "${STACK_DIR}" -f "${STACK_DIR}/docker-compose.yml")
74+
[[ ! -f "${STACK_DIR}/.env" ]] || COMPOSE+=(--env-file "${STACK_DIR}/.env")
75+
76+
compose() {
77+
"${COMPOSE[@]}" "$@"
78+
}
79+
80+
sha256_file() {
81+
sha256sum < "$1"
82+
}
83+
84+
validate_certificate() {
85+
local cert_public_key
86+
local key_public_key
87+
88+
[[ "${SLACK_WEBHOOK:-}" == https://* ]] || fail 'SLACK_WEBHOOK is not configured'
89+
[[ -r "${SOURCE_CERT}" ]] || fail "certificate is not readable: ${SOURCE_CERT}"
90+
[[ -r "${SOURCE_KEY}" ]] || fail "private key is not readable: ${SOURCE_KEY}"
91+
[[ "$(grep -cF -- '-----BEGIN CERTIFICATE-----' "${SOURCE_CERT}" || true)" -ge 2 ]] || \
92+
fail 'certificate does not contain a full chain'
93+
94+
openssl x509 -in "${SOURCE_CERT}" -noout -checkend 86400 >/dev/null 2>&1 || \
95+
fail 'certificate is expired or expires within 24 hours'
96+
openssl x509 -in "${SOURCE_CERT}" -noout -checkhost "${EXPECTED_HOST}" >/dev/null 2>&1 || \
97+
fail "certificate does not cover ${EXPECTED_HOST}"
98+
openssl verify -purpose sslserver -untrusted "${SOURCE_CERT}" "${SOURCE_CERT}" >/dev/null 2>&1 || \
99+
fail 'certificate chain is not trusted for TLS server use'
100+
openssl pkey -in "${SOURCE_KEY}" -check -noout </dev/null >/dev/null 2>&1 || \
101+
fail 'private key is invalid or encrypted'
102+
103+
cert_public_key="$(openssl x509 -in "${SOURCE_CERT}" -pubkey -noout |
104+
openssl pkey -pubin -outform DER 2>/dev/null | sha256sum)"
105+
key_public_key="$(openssl pkey -in "${SOURCE_KEY}" -pubout -outform DER </dev/null 2>/dev/null |
106+
sha256sum)"
107+
[[ "${cert_public_key}" == "${key_public_key}" ]] || fail 'certificate and private key do not match'
108+
}
109+
110+
secrets_are_current() {
111+
[[ -f "${TARGET_CERT}" && -f "${TARGET_KEY}" ]] || return 1
112+
[[ "$(sha256_file "${SOURCE_CERT}")" == "$(sha256_file "${TARGET_CERT}")" ]] || return 1
113+
[[ "$(sha256_file "${SOURCE_KEY}")" == "$(sha256_file "${TARGET_KEY}")" ]]
114+
}
115+
116+
main() {
117+
validate_certificate
118+
install -d -o root -g root -m 0755 "${CERT_DIR}"
119+
120+
if secrets_are_current && [[ ! -e "${RESTART_REQUIRED}" ]]; then
121+
printf '%s\n' 'TLS certificate is already current; Traefik was not recreated.'
122+
return
123+
fi
124+
125+
ERROR_MESSAGE='Compose configuration validation failed'
126+
compose config --quiet
127+
128+
if ! secrets_are_current; then
129+
ERROR_MESSAGE='could not copy the renewed TLS certificate'
130+
install -o root -g root -m 0600 /dev/null "${RESTART_REQUIRED}"
131+
STAGED_CERT="${TARGET_CERT}.new"
132+
STAGED_KEY="${TARGET_KEY}.new"
133+
install -o root -g root -m 0644 "${SOURCE_CERT}" "${STAGED_CERT}"
134+
install -o root -g root -m 0600 "${SOURCE_KEY}" "${STAGED_KEY}"
135+
mv -f "${STAGED_KEY}" "${TARGET_KEY}"
136+
STAGED_KEY=""
137+
mv -f "${STAGED_CERT}" "${TARGET_CERT}"
138+
STAGED_CERT=""
139+
fi
140+
141+
ERROR_MESSAGE='Traefik failed to restart or become healthy'
142+
compose up -d --no-deps --no-build --pull never --force-recreate \
143+
--wait --wait-timeout "${HEALTHCHECK_TIMEOUT}" traefik
144+
145+
rm -f -- "${RESTART_REQUIRED}"
146+
printf '%s\n' 'TLS certificate copied; Traefik is healthy.'
147+
}
148+
149+
main "$@"

‎scripts/local-cert-hook.env.example‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Installed as /etc/default/local-cert-hook with mode 0600.
2+
SLACK_WEBHOOK=''
3+
4+
# The hostname used to validate the certificate and the live TLS endpoint.
5+
EXPECTED_HOST='archivesspace.lib.lehigh.edu'
6+
7+
# These defaults can be overridden when the hook is reused by another stack.
8+
# SOURCE_CERT='/etc/ssl/certs/le/lib.lehigh.edu.pem'
9+
# SOURCE_KEY='/etc/ssl/private/le/lib.lehigh.edu.key'
10+
# STACK_DIR='/opt/archivesspace'
11+
# HEALTHCHECK_TIMEOUT='60'
12+
# ALERT_CONTEXT='ArchivesSpace TLS certificate rollout'

0 commit comments

Comments
 (0)