Skip to content

Commit f63beca

Browse files
authored
Merge pull request #35 from persys-dev/Feat/Vault-manager-crashproof
Feat/vault manager crashproof
2 parents e8ebf88 + 1e2fa90 commit f63beca

7 files changed

Lines changed: 357 additions & 61 deletions

File tree

vault-manager/Dockerfile

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o
1010

1111
FROM alpine:latest
1212

13-
RUN apk add --no-cache ca-certificates \
14-
&& addgroup -S app \
15-
&& adduser -S -G app app
13+
RUN apk add --no-cache ca-certificates
1614

1715
COPY --from=build /out/vault-manager /usr/local/bin/vault-manager
1816

19-
USER app
2017
ENTRYPOINT ["vault-manager"]

vault-manager/README.md

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ vault-manager/
4747

4848
```bash
4949
cd vault-manager
50-
go run ./cmd/vault-manager --vault-addr=http://localhost:8200
50+
go run ./cmd/main.go --vault-addr=http://localhost:8200
5151
```
5252

5353
On first run against an uninitialized Vault, the unseal key and root token
@@ -56,14 +56,14 @@ afterward. On subsequent runs against an already-initialized Vault, set
5656
`VAULT_ROOT_TOKEN` in the environment instead:
5757

5858
```bash
59-
VAULT_ROOT_TOKEN=hvs.xxxxx go run ./cmd/vault-manager --vault-addr=http://localhost:8200
59+
VAULT_ROOT_TOKEN=hvs.xxxxx go run ./cmd/main.go --vault-addr=http://localhost:8200
6060
```
6161

6262
To provision once with root and then drop root privileges for the life of
6363
the process:
6464

6565
```bash
66-
go run ./cmd/vault-manager --vault-addr=http://localhost:8200 --secure
66+
go run ./cmd/main.go --vault-addr=http://localhost:8200 --secure
6767
```
6868

6969
## CLI flags
@@ -78,6 +78,7 @@ go run ./cmd/vault-manager --vault-addr=http://localhost:8200 --secure
7878
| `--manager-role` | `vault-manager-bootstrap` | AppRole name used for the `--secure` bootstrap handoff |
7979
| `--manager-policy` | `vault-manager-bootstrap-policy` | ACL policy name for the bootstrap manager AppRole |
8080
| `--services` | `persys-gateway,persys-scheduler,persysctl,compute-agent,persys-forgery,persys-services,persys-automation,persys-intelligence,persys-sdk` | Comma-separated list of services to provision |
81+
| `--bootstrap-file` | `/var/lib/persys/vault/bootstrap.json` | Persistent recovery credentials (unseal keys + auth) |
8182
| `--secure` | `false` | Provision a bootstrap AppRole and revoke the root token after setup |
8283

8384
## Environment variables
@@ -125,6 +126,27 @@ logs (method, status code, duration, and any error) for each call.
125126
In docker compose, this is used by the `vault-manager` profile in
126127
`infra/docker/docker-compose.yml`.
127128

129+
130+
## Restart / recovery
131+
132+
Unseal keys and auth credentials are written to `--bootstrap-file`
133+
(default `/var/lib/persys/vault/bootstrap.json`) on first init.
134+
135+
After `docker compose down` (without `-v`) and `up` again:
136+
137+
1. Vault comes back **sealed**.
138+
2. vault-manager loads the bootstrap file from the `vault_manager_data` volume.
139+
3. It unseals Vault with the stored keys, then authenticates (manager AppRole or root token).
140+
141+
**Requirements:**
142+
143+
- Named volume mounted at `/var/lib/persys/vault/` (as in compose).
144+
- Do **not** use `docker compose down -v` unless you intend to wipe recovery state.
145+
- The image entrypoint chowns the volume so the non-root `app` user can write `bootstrap.json`.
146+
147+
If the bootstrap file is missing on an already-initialized Vault, set
148+
`VAULT_ROOT_TOKEN` once; after a successful run the file is rewritten.
149+
128150
## Operational notes
129151

130152
- Single key-share initialization (`secret_shares: 1`, `secret_threshold:

vault-manager/cmd/main.go

Lines changed: 135 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@
22
// and unseals Vault if needed, sets up the PKI CA chain, provisions
33
// per-service AppRoles and policies, then serves a gRPC API so other
44
// services can fetch or rotate their credentials at runtime.
5+
//
6+
// On first run the unseal key(s) and auth credentials are written to
7+
// --bootstrap-file so subsequent restarts (of vault-manager or of Vault
8+
// itself) can recover without operator intervention.
59
package main
610

711
import (
12+
"errors"
813
"fmt"
914
"os"
1015
"os/signal"
@@ -15,6 +20,7 @@ import (
1520
"github.com/sirupsen/logrus"
1621

1722
"github.com/persys-dev/persys-cloud/vault-manager/internal/approle"
23+
"github.com/persys-dev/persys-cloud/vault-manager/internal/bootstrap"
1824
"github.com/persys-dev/persys-cloud/vault-manager/internal/config"
1925
"github.com/persys-dev/persys-cloud/vault-manager/internal/pki"
2026
"github.com/persys-dev/persys-cloud/vault-manager/internal/policy"
@@ -28,34 +34,23 @@ func main() {
2834
config.Log.Fatal("no valid services found in --services")
2935
}
3036

31-
baseClient, err := vaultclient.New(cfg.VaultAddr, "")
32-
if err != nil {
33-
config.Log.Fatal(err)
34-
}
35-
vaultclient.WaitUntilReady(baseClient)
37+
// Sealed Vault is fine — we unseal from bootstrap state next.
38+
vaultclient.WaitUntilReady(cfg.VaultAddr)
3639

37-
rootToken, err := bootstrapOrUnseal(cfg)
40+
workClient, state, err := recoverOrBootstrap(cfg)
3841
if err != nil {
3942
config.Log.Fatal(err)
4043
}
4144

42-
rootClient, err := vaultclient.New(cfg.VaultAddr, rootToken)
43-
if err != nil {
45+
if err := provision(workClient, cfg); err != nil {
4446
config.Log.Fatal(err)
4547
}
4648

47-
workClient := rootClient
48-
if cfg.Secure {
49-
config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token")
50-
workClient, err = vaultclient.SwitchToSecure(rootClient, cfg)
51-
if err != nil {
52-
config.Log.Fatal(err)
53-
}
54-
}
55-
56-
if err := provision(workClient, cfg); err != nil {
57-
config.Log.Fatal(err)
49+
// Persist latest state after successful provision.
50+
if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil {
51+
config.Log.Fatalf("save bootstrap state to %s: %v", cfg.BootstrapFile, err)
5852
}
53+
config.Log.Printf("Bootstrap state saved to %s", cfg.BootstrapFile)
5954

6055
secrets, err := approle.GatherSecrets(workClient, cfg)
6156
if err != nil {
@@ -77,39 +72,141 @@ func main() {
7772
waitForShutdown()
7873
}
7974

80-
// bootstrapOrUnseal initializes and unseals Vault if it hasn't been set up
81-
// yet, then returns the root token to use for provisioning: the freshly
82-
// generated one, or VAULT_ROOT_TOKEN if Vault was already initialized.
83-
func bootstrapOrUnseal(cfg *config.Config) (string, error) {
75+
// recoverOrBootstrap is the restart-safe entry point.
76+
//
77+
// 1. Load bootstrap file if present.
78+
// 2. Uninitialized Vault → init, unseal, persist keys + root token.
79+
// 3. Initialized + sealed → unseal with stored keys.
80+
// 4. Authenticate: manager AppRole (preferred) → stored root token → VAULT_ROOT_TOKEN.
81+
// 5. --secure without manager creds → hand off, revoke root, persist manager creds.
82+
func recoverOrBootstrap(cfg *config.Config) (*vault.Client, *bootstrap.State, error) {
83+
state, err := bootstrap.Load(cfg.BootstrapFile)
84+
if err != nil && !errors.Is(err, bootstrap.ErrNotFound) {
85+
return nil, nil, fmt.Errorf("load bootstrap state from %s: %w", cfg.BootstrapFile, err)
86+
}
87+
if state == nil {
88+
state = &bootstrap.State{}
89+
config.Log.Printf("No bootstrap state at %s (first run or missing volume)", cfg.BootstrapFile)
90+
} else {
91+
config.Log.Printf("Loaded bootstrap state from %s (unseal_keys=%d manager=%v root=%v)",
92+
cfg.BootstrapFile, len(state.UnsealKeys), state.HasManagerCreds(), state.RootToken != "")
93+
}
94+
8495
initialized, err := vaultclient.IsInitialized(cfg.VaultAddr)
8596
if err != nil {
86-
return "", err
97+
return nil, nil, err
8798
}
8899

89100
if !initialized {
90-
config.Log.Println("Vault not initialized. Initializing...")
91-
initResult, err := vaultclient.Initialize(cfg.VaultAddr)
101+
return firstTimeInit(cfg, state)
102+
}
103+
104+
if err := vaultclient.EnsureUnsealed(cfg.VaultAddr, state.UnsealKeys); err != nil {
105+
return nil, nil, err
106+
}
107+
108+
client, err := authenticate(cfg, state)
109+
if err != nil {
110+
return nil, nil, err
111+
}
112+
113+
if cfg.Secure && !state.HasManagerCreds() {
114+
config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token")
115+
handoff, err := vaultclient.SwitchToSecure(client, cfg)
92116
if err != nil {
93-
return "", err
117+
return nil, nil, err
94118
}
95-
fmt.Println("Vault initialized credentials (store securely):")
96-
fmt.Printf("unseal_key: %s\n", initResult.UnsealKey)
97-
fmt.Printf("root_token: %s\n", initResult.RootToken)
98-
if err := vaultclient.Unseal(cfg.VaultAddr, initResult.UnsealKey); err != nil {
99-
return "", err
119+
state.ManagerRoleID = handoff.RoleID
120+
state.ManagerSecretID = handoff.SecretID
121+
state.RootToken = ""
122+
client = handoff.Client
123+
}
124+
125+
return client, state, nil
126+
}
127+
128+
func firstTimeInit(cfg *config.Config, state *bootstrap.State) (*vault.Client, *bootstrap.State, error) {
129+
config.Log.Println("Vault not initialized. Initializing...")
130+
initResult, err := vaultclient.Initialize(cfg.VaultAddr)
131+
if err != nil {
132+
return nil, nil, err
133+
}
134+
135+
fmt.Println("Vault initialized credentials (also saved to bootstrap file):")
136+
fmt.Printf("unseal_keys: %v\n", initResult.UnsealKeys)
137+
fmt.Printf("root_token: %s\n", initResult.RootToken)
138+
139+
if err := vaultclient.UnsealAll(cfg.VaultAddr, initResult.UnsealKeys); err != nil {
140+
return nil, nil, err
141+
}
142+
config.Log.Println("Vault initialized and unsealed.")
143+
144+
state.UnsealKeys = initResult.UnsealKeys
145+
state.RootToken = initResult.RootToken
146+
147+
// Persist immediately so a crash between init and provision is recoverable.
148+
if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil {
149+
return nil, nil, fmt.Errorf("save bootstrap state after init to %s: %w", cfg.BootstrapFile, err)
150+
}
151+
config.Log.Printf("Bootstrap state saved to %s", cfg.BootstrapFile)
152+
153+
client, err := vaultclient.New(cfg.VaultAddr, initResult.RootToken)
154+
if err != nil {
155+
return nil, nil, err
156+
}
157+
158+
if cfg.Secure {
159+
config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token")
160+
handoff, err := vaultclient.SwitchToSecure(client, cfg)
161+
if err != nil {
162+
return nil, nil, err
163+
}
164+
state.ManagerRoleID = handoff.RoleID
165+
state.ManagerSecretID = handoff.SecretID
166+
state.RootToken = ""
167+
client = handoff.Client
168+
169+
if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil {
170+
return nil, nil, fmt.Errorf("save bootstrap state after secure handoff: %w", err)
100171
}
101-
config.Log.Println("Vault initialized and unsealed.")
102-
return initResult.RootToken, nil
103172
}
104173

105-
rootToken := strings.TrimSpace(os.Getenv("VAULT_ROOT_TOKEN"))
174+
return client, state, nil
175+
}
176+
177+
// authenticate picks the best available credential source.
178+
//
179+
// Priority:
180+
// 1. Manager AppRole from bootstrap file (restart after --secure)
181+
// 2. Root token from bootstrap file
182+
// 3. VAULT_ROOT_TOKEN environment variable
183+
func authenticate(cfg *config.Config, state *bootstrap.State) (*vault.Client, error) {
184+
if state.HasManagerCreds() {
185+
config.Log.Println("Authenticating with stored manager AppRole credentials")
186+
client, err := vaultclient.LoginAppRole(cfg.VaultAddr, state.ManagerRoleID, state.ManagerSecretID)
187+
if err != nil {
188+
return nil, fmt.Errorf("manager AppRole login: %w", err)
189+
}
190+
return client, nil
191+
}
192+
193+
rootToken := strings.TrimSpace(state.RootToken)
106194
if rootToken == "" {
107-
return "", fmt.Errorf("VAULT_ROOT_TOKEN required when Vault is already initialized")
195+
rootToken = strings.TrimSpace(os.Getenv("VAULT_ROOT_TOKEN"))
108196
}
109-
return rootToken, nil
197+
if rootToken == "" {
198+
return nil, fmt.Errorf(
199+
"vault is initialized but no credentials available: "+
200+
"ensure %s contains unseal_keys and root_token/manager creds, "+
201+
"or set VAULT_ROOT_TOKEN (file missing usually means the volume was not persisted)",
202+
cfg.BootstrapFile,
203+
)
204+
}
205+
206+
config.Log.Println("Authenticating with root token")
207+
return vaultclient.New(cfg.VaultAddr, rootToken)
110208
}
111209

112-
// provision ensures the PKI chain, service policies, and AppRoles all exist.
113210
func provision(client *vault.Client, cfg *config.Config) error {
114211
if err := pki.Ensure(client, cfg); err != nil {
115212
return err
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Package bootstrap persists the credentials needed to recover after a
2+
// vault-manager (or Vault itself) restart: unseal keys and either a root
3+
// token or the bootstrap-manager AppRole credentials.
4+
package bootstrap
5+
6+
import (
7+
"encoding/json"
8+
"errors"
9+
"os"
10+
"path/filepath"
11+
)
12+
13+
// State is the on-disk recovery record written after a successful bootstrap.
14+
type State struct {
15+
UnsealKeys []string `json:"unseal_keys"`
16+
17+
// RootToken is kept when running without --secure. Cleared after a
18+
// successful --secure handoff so the file never holds a live root token.
19+
RootToken string `json:"root_token,omitempty"`
20+
21+
// Manager AppRole credentials used when --secure is enabled (and on
22+
// subsequent restarts of a previously secured deployment).
23+
ManagerRoleID string `json:"manager_role_id,omitempty"`
24+
ManagerSecretID string `json:"manager_secret_id,omitempty"`
25+
}
26+
27+
// ErrNotFound is returned by Load when the bootstrap file does not exist.
28+
var ErrNotFound = errors.New("bootstrap state file not found")
29+
30+
// Load reads and decodes the bootstrap state from path.
31+
// Returns ErrNotFound if the file does not exist.
32+
func Load(path string) (*State, error) {
33+
b, err := os.ReadFile(path)
34+
if err != nil {
35+
if os.IsNotExist(err) {
36+
return nil, ErrNotFound
37+
}
38+
return nil, err
39+
}
40+
41+
var s State
42+
if err := json.Unmarshal(b, &s); err != nil {
43+
return nil, err
44+
}
45+
return &s, nil
46+
}
47+
48+
// Save writes state to path atomically (tmp + rename) with restrictive perms.
49+
func Save(path string, s *State) error {
50+
dir := filepath.Dir(path)
51+
if err := os.MkdirAll(dir, 0700); err != nil {
52+
return err
53+
}
54+
55+
tmp := path + ".tmp"
56+
b, err := json.MarshalIndent(s, "", " ")
57+
if err != nil {
58+
return err
59+
}
60+
if err := os.WriteFile(tmp, b, 0600); err != nil {
61+
return err
62+
}
63+
return os.Rename(tmp, path)
64+
}
65+
66+
// HasManagerCreds reports whether the state holds usable manager AppRole credentials.
67+
func (s *State) HasManagerCreds() bool {
68+
return s != nil && s.ManagerRoleID != "" && s.ManagerSecretID != ""
69+
}
70+
71+
// HasUnsealKeys reports whether the state holds at least one unseal key.
72+
func (s *State) HasUnsealKeys() bool {
73+
return s != nil && len(s.UnsealKeys) > 0
74+
}

0 commit comments

Comments
 (0)