From 2af4140e11fd14c395cc7a2af22888661bc61e4e Mon Sep 17 00:00:00 2001 From: Bugs5382 <12115015+Bugs5382@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:40:26 -0400 Subject: [PATCH 1/3] fix(init): read the state-key mode from the volume after the first boot A node whose state_key.mode came from its machine config and differed from the image's build-time default could not come back from any reboot. init resolved the mode before unlock from the ESP config stage only, and the stage exists only on the first boot, so every later boot fell back to the build-time mode and opened the volume with the wrong protector. Later boots now derive the mode from the LUKS2 header the volume was sealed with: a cryptos-tpm2 token means tpm, a cryptos-kms token means kms (with the endpoint and trust it carries), and no protector token means nodeid. The first boot still takes it from the staged config, else the build-time default. The multi-token TPM header left by an upgrade still reads as tpm. The KMS token carried no type or keyslots, which LUKS2 requires, so cryptsetup refused to import it and kms mode could not finish a first boot. It now has type cryptos-kms and the state keyslot. The mode is fixed at install. ApplyConfig on a running node and the re-provision installer refuse a config naming another state_key.mode with FailedPrecondition, and a boot whose persisted config disagrees logs it and keeps the sealed mode. Signed-off-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com> --- internal/config/config.go | 14 +- internal/config/statekey_sealed_test.go | 51 ++++ internal/init/configload.go | 11 +- internal/init/kms_protector.go | 17 +- internal/init/maintenance.go | 4 +- internal/init/reprovision.go | 5 + internal/init/reprovision_test.go | 25 ++ internal/init/run.go | 52 ++-- internal/init/stateunlock.go | 152 ++++++++++ internal/init/stateunlock_test.go | 377 ++++++++++++++++++++++++ internal/node/apply_statekey_test.go | 76 +++++ internal/node/providers.go | 18 ++ 12 files changed, 757 insertions(+), 45 deletions(-) create mode 100644 internal/config/statekey_sealed_test.go create mode 100644 internal/init/stateunlock.go create mode 100644 internal/init/stateunlock_test.go create mode 100644 internal/node/apply_statekey_test.go diff --git a/internal/config/config.go b/internal/config/config.go index dd07e76..457063b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -144,12 +144,24 @@ type Config struct { // StateKey selects the protector for the encrypted state-partition key. Mode is // "" (build-time default) | "nodeid" | "tpm" | "kms". It is carried in the proto // MachineConfig so the choice survives ApplyConfig and reaches an installed node -// (the maintenance installer reconstructs the staged YAML from the proto). +// (the maintenance installer reconstructs the staged YAML from the proto). The +// mode takes effect when the state volume is first sealed; every later boot +// reads it from the volume. type StateKey struct { Mode string `yaml:"mode"` KMS *KmsStateKey `yaml:"kms"` } +// CheckSealed refuses a mode other than sealed, the mode the node's state +// volume was sealed with at install. An empty mode asks for nothing and passes. +func (sk StateKey) CheckSealed(sealed string) error { + if sk.Mode == "" || sk.Mode == sealed { + return nil + } + return fmt.Errorf("config: state_key.mode %q: this node's state volume is sealed in %q mode, which is fixed at install; reinstall the node to change it", + sk.Mode, sealed) +} + // KmsStateKey configures the envelope-encryption KMS that seals/unseals the // state key. Only first-boot provisioning reads this from the machine config; // later boots recover the endpoint and sealed blob from the LUKS2 header token. diff --git a/internal/config/statekey_sealed_test.go b/internal/config/statekey_sealed_test.go new file mode 100644 index 0000000..035c454 --- /dev/null +++ b/internal/config/statekey_sealed_test.go @@ -0,0 +1,51 @@ +package config + +/* +Apache License 2.0 + +Copyright 2026 Shane + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +import ( + "strings" + "testing" +) + +// The state-key mode is fixed when the volume is sealed at install. A config +// may repeat it or leave it empty, but may not name another mode. +func TestStateKeyCheckSealed(t *testing.T) { + cases := []struct { + name, configured, sealed string + wantErr bool + }{ + {"empty keeps the sealed mode", "", StateKeyModeTPM, false}, + {"same mode", StateKeyModeNodeID, StateKeyModeNodeID, false}, + {"tpm on a nodeid volume", StateKeyModeTPM, StateKeyModeNodeID, true}, + {"nodeid on a tpm volume", StateKeyModeNodeID, StateKeyModeTPM, true}, + {"kms on a tpm volume", StateKeyModeKMS, StateKeyModeTPM, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := StateKey{Mode: tc.configured}.CheckSealed(tc.sealed) + if (err != nil) != tc.wantErr { + t.Fatalf("CheckSealed = %v, want error %v", err, tc.wantErr) + } + if err != nil && (!strings.Contains(err.Error(), tc.configured) || !strings.Contains(err.Error(), tc.sealed)) { + t.Errorf("error %q should name both modes", err) + } + }) + } +} diff --git a/internal/init/configload.go b/internal/init/configload.go index 8a3bd04..ab24d61 100644 --- a/internal/init/configload.go +++ b/internal/init/configload.go @@ -100,12 +100,11 @@ func loadOrSeedConfig(store *config.FileStore, stage espStageAccessors) (*config return nil, fmt.Errorf("%w: no persisted config on an installed node", errEnterMaintenance) } -// preUnlockStateKey resolves the state-key selection readable before the -// encrypted volume is open: the ESP stage (present on first boot with the -// operator's config). It parses only the state_key section, tolerating an -// absent or unparseable stage by returning the zero StateKey — the caller then -// falls back to the build-time default and the volume opens with the token on a -// later boot. It never fails the boot: pre-unlock config is best-effort. +// preUnlockStateKey reads the first boot's state-key selection from the ESP +// stage, which carries the operator's config before the volume exists. It +// parses only the state_key section, tolerating an absent or unparseable stage +// by returning the zero StateKey, so the caller falls back to the build-time +// default. Later boots never call it: they read the mode from the volume. func preUnlockStateKey(stage espStageAccessors) config.StateKey { if stage.stageReader == nil { return config.StateKey{} diff --git a/internal/init/kms_protector.go b/internal/init/kms_protector.go index 59bd8e5..8fc2a89 100644 --- a/internal/init/kms_protector.go +++ b/internal/init/kms_protector.go @@ -24,6 +24,7 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "github.com/CryptOS-PKI/cryptos/internal/config" "github.com/CryptOS-PKI/cryptos/internal/kms" @@ -42,13 +43,19 @@ type kmsProtector struct { newProvider func(endpoint string, trustPEM []byte) (kms.Provider, error) } +// kmsTokenType is the LUKS2 token type of the KMS protector's token. The type +// is also how a later boot knows the volume was sealed in kms mode. +const kmsTokenType = "cryptos-kms" + // kmsToken is the JSON persisted in the LUKS2 header token. It carries the // endpoint and trust bundle so later boots can rebuild the Provider without the -// machine config, plus the sealed DEK blob. +// machine config, plus the sealed DEK blob. LUKS2 requires type and keyslots. type kmsToken struct { - Endpoint string `json:"endpoint"` - TrustPEM string `json:"trust_pem"` - Sealed []byte `json:"sealed"` + Type string `json:"type"` + Keyslots []string `json:"keyslots"` + Endpoint string `json:"endpoint"` + TrustPEM string `json:"trust_pem"` + Sealed []byte `json:"sealed"` } // newKMSProtector builds a kmsProtector from the machine-config KMS settings. @@ -99,6 +106,8 @@ func (p *kmsProtector) ProvisionKey(ctx context.Context) (key, token []byte, err return nil, nil, fmt.Errorf("kms: seal: %w", err) } token, err = json.Marshal(kmsToken{ + Type: kmsTokenType, + Keyslots: []string{strconv.Itoa(stateKeyslot)}, Endpoint: p.endpoint, TrustPEM: string(p.trustPEM), Sealed: sealed, diff --git a/internal/init/maintenance.go b/internal/init/maintenance.go index cb6a2a4..2a0bf13 100644 --- a/internal/init/maintenance.go +++ b/internal/init/maintenance.go @@ -140,7 +140,7 @@ func runMaintenance(ctx context.Context) error { // The listener setup mirrors runMaintenance: self-signed cert, client auth OFF // (Talos --insecure), no-op auditor. It parks until a shutdown signal or a // successful re-provision (which triggers reboot), then returns so PID 1 reboots. -func runReprovisionMaintenance(ctx context.Context, cfgStore *config.FileStore) error { +func runReprovisionMaintenance(ctx context.Context, cfgStore *config.FileStore, sealedMode string) error { if err := netlink.BringUpLoopback(); err != nil { return err } @@ -154,7 +154,7 @@ func runReprovisionMaintenance(ctx context.Context, cfgStore *config.FileStore) // client before tearing down the connection, then this function returns and // PID 1 reboots into the ceremony. rebootCh := make(chan struct{}, 1) - rp := &reprovisioner{store: cfgStore, rebootCh: rebootCh} + rp := &reprovisioner{store: cfgStore, rebootCh: rebootCh, sealedMode: sealedMode} srv, err := cgrpc.NewMaintenance(cgrpc.ServerConfig{ TLSConfig: MaintenanceServerTLSConfig(serverCert), diff --git a/internal/init/reprovision.go b/internal/init/reprovision.go index 46cbd68..00d403f 100644 --- a/internal/init/reprovision.go +++ b/internal/init/reprovision.go @@ -50,6 +50,8 @@ import ( type reprovisioner struct { store *config.FileStore rebootCh chan struct{} + // sealedMode is the state-key mode the kept state volume was sealed with. + sealedMode string } // Install validates cfg, persists it to the mounted state store, signals a @@ -60,6 +62,9 @@ func (r *reprovisioner) Install(_ context.Context, cfg *cryptosv1.MachineConfig) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "reprovision: parse: %v", err) } + if err := parsed.StateKey.CheckSealed(r.sealedMode); err != nil { + return nil, status.Errorf(codes.FailedPrecondition, "reprovision: %v", err) + } if err := parsed.Validate(); err != nil { return nil, status.Errorf(codes.InvalidArgument, "reprovision: validate: %v", err) } diff --git a/internal/init/reprovision_test.go b/internal/init/reprovision_test.go index 5bd50fa..1aed192 100644 --- a/internal/init/reprovision_test.go +++ b/internal/init/reprovision_test.go @@ -25,6 +25,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + cryptosv1 "github.com/CryptOS-PKI/api/go/cryptos/v1" "github.com/CryptOS-PKI/cryptos/internal/config" ) @@ -114,3 +115,27 @@ func TestReprovisioner_InvalidConfig(t *testing.T) { // expected: no reboot signal } } + +// Re-provisioning keeps the state volume, so it keeps the mode the volume was +// sealed with. A config naming another mode is refused, not persisted, and +// does not reboot. +func TestReprovisioner_RefusesStateKeyModeChange(t *testing.T) { + store := config.NewFileStore(t.TempDir()) + rebootCh := make(chan struct{}, 1) + rp := &reprovisioner{store: store, rebootCh: rebootCh, sealedMode: config.StateKeyModeNodeID} + + cfg := validMachineConfig() + cfg.StateKey = &cryptosv1.StateKey{Mode: config.StateKeyModeTPM} + _, err := rp.Install(context.Background(), cfg) + if got := status.Code(err); got != codes.FailedPrecondition { + t.Fatalf("Install code = %v, want FailedPrecondition (err: %v)", got, err) + } + if _, _, ok, _ := store.Read(); ok { + t.Error("config persisted after a refused re-provision") + } + select { + case <-rebootCh: + t.Error("reboot signalled after a refused re-provision") + default: + } +} diff --git a/internal/init/run.go b/internal/init/run.go index dd0435d..0ad95a0 100644 --- a/internal/init/run.go +++ b/internal/init/run.go @@ -210,45 +210,30 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) { } begin("state volume") - // 3. State-key + Root-key backends (TPM-sealed by default; nodeID/software - // or KMS-wrapped for the TPM-less variants). The state-key selection is - // resolved pre-unlock: on first boot the ESP-staged config (readable before - // the volume opens) supplies the kms endpoint for ProvisionKey; on later - // boots the staged config is gone and the build-time StateKeyMode default - // applies (RecoverKey then reads the endpoint from the LUKS2 token, not - // config). The effective mode is the staged config's when set, else the - // build-time default. - sk := preUnlockStateKey(realESPStageAccessors()) - mode := StateKeyMode - if sk.Mode != "" { - mode = sk.Mode - } - protector, rootBackend, closeBackends, tpmState, err := newStateKeyBackends(mode, sk) - if err != nil { - return err - } - defer closeBackends() - log.Printf("state key mode: %s", protector.Name()) - - // 4. Open (or first-boot-format) the encrypted state volume. Resolve the - // state partition by its GPT name via sysfs (the image has no udev, so the - // by-partlabel symlinks never exist); devtmpfs has created the /dev node. - // First-boot is decided from the partition itself (!IsLUKS), not from config, - // because config does not exist yet. + // 3. Resolve the state partition by its GPT name via sysfs (the image has no + // udev, so the by-partlabel symlinks never exist); devtmpfs has created the + // /dev node. First-boot is decided from the partition itself (!IsLUKS), not + // from config, because config does not exist yet. stateDevice, err := resolveStateDevice(StateLabel) if err != nil { return err } paths.Device = stateDevice dev := &luks.Device{Path: paths.Device, Runner: &luks.ExecRunner{Binary: cryptsetupBinary}} - firstBoot := !dev.IsLUKS(ctx) - vol, err := OpenStateVolume(ctx, StateVolumeConfig{ - Protector: protector, Device: dev, MappedName: StateMappedName, - TokenID: StateTokenID, FirstBoot: firstBoot, - }) + + // 4. State-key + Root-key backends (TPM-sealed by default; nodeID/software + // or KMS-wrapped for the TPM-less variants), then open (or first-boot-format) + // the encrypted state volume with them. + st, err := stateUnlocker{ + device: dev, stage: realESPStageAccessors(), + buildDefault: StateKeyMode, newBackends: newStateKeyBackends, + }.unlock(ctx) if err != nil { return err } + defer st.close() + log.Printf("state key mode: %s", st.protector.Name()) + mode, rootBackend, tpmState, vol, firstBoot := st.mode, st.root, st.tpmState, st.vol, st.firstBoot if firstBoot { if err := mkfsExt4(vol.Path); err != nil { return err @@ -292,10 +277,13 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) { // which persists the applied config to the mounted state and reboots // into the ceremony, rather than the bare-disk installer. log.Printf("REPROVISION: %v", err) - return runReprovisionMaintenance(ctx, cfgStore) + return runReprovisionMaintenance(ctx, cfgStore, mode) } return err } + if err := cfg.StateKey.CheckSealed(mode); err != nil { + log.Printf("state key: %v; using the sealed mode", err) + } done() begin("network") @@ -441,7 +429,7 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) { Identity: node.NewIdentityProvider(store), Status: statusProv, Ceremony: eng, - ConfigStore: node.NewConfigStore(cfgStore).WithIssuer(issuerFunc), + ConfigStore: node.NewConfigStore(cfgStore).WithIssuer(issuerFunc).WithSealedStateKeyMode(mode), } } diff --git a/internal/init/stateunlock.go b/internal/init/stateunlock.go new file mode 100644 index 0000000..eae2848 --- /dev/null +++ b/internal/init/stateunlock.go @@ -0,0 +1,152 @@ +package init + +/* +Apache License 2.0 + +Copyright 2026 Shane + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "slices" + + cryptosv1 "github.com/CryptOS-PKI/api/go/cryptos/v1" + "github.com/CryptOS-PKI/cryptos/internal/ceremony" + "github.com/CryptOS-PKI/cryptos/internal/config" + "github.com/CryptOS-PKI/cryptos/internal/storage/luks" +) + +// stateBackendFactory builds the state-key protector and Root-key backend for +// a state-key mode. newStateKeyBackends is the production factory. +type stateBackendFactory func(mode string, sk config.StateKey) (StateKeyProtector, ceremony.RootKeyBackend, func(), cryptosv1.TpmState, error) + +// stateUnlocker opens (or, on first boot, formats) the encrypted state volume +// with the protector for the node's state-key mode. +type stateUnlocker struct { + device *luks.Device + stage espStageAccessors + // buildDefault is the image's build-time mode, used when first-boot config + // selects none. + buildDefault string + newBackends stateBackendFactory +} + +// unlockedState is an open state volume and the backends it was opened with. +// The caller owns close. +type unlockedState struct { + mode string + firstBoot bool + vol *luks.Volume + protector StateKeyProtector + root ceremony.RootKeyBackend + close func() + tpmState cryptosv1.TpmState +} + +func (u stateUnlocker) unlock(ctx context.Context) (*unlockedState, error) { + firstBoot := !u.device.IsLUKS(ctx) + mode, sk, err := u.stateKey(ctx, firstBoot) + if err != nil { + return nil, err + } + protector, root, closeFn, tpmState, err := u.newBackends(mode, sk) + if err != nil { + return nil, err + } + vol, err := OpenStateVolume(ctx, StateVolumeConfig{ + Protector: protector, Device: u.device, MappedName: StateMappedName, + TokenID: StateTokenID, FirstBoot: firstBoot, + }) + if err != nil { + closeFn() + return nil, err + } + return &unlockedState{ + mode: mode, firstBoot: firstBoot, vol: vol, protector: protector, + root: root, close: closeFn, tpmState: tpmState, + }, nil +} + +// stateKey resolves the state-key selection before the volume is open. On +// first boot it comes from the ESP-staged config, else the build-time default. +// Every later boot reads it from the volume's LUKS2 header, which records how +// the volume was sealed; the config (inside the volume) is not readable yet, +// and the stage is gone. +func (u stateUnlocker) stateKey(ctx context.Context, firstBoot bool) (string, config.StateKey, error) { + if firstBoot { + sk := preUnlockStateKey(u.stage) + if sk.Mode == "" { + return u.buildDefault, sk, nil + } + return sk.Mode, sk, nil + } + tokens, err := u.device.Tokens(ctx) + if err != nil { + return "", config.StateKey{}, fmt.Errorf("init: read the state volume's tokens: %w", err) + } + sk, err := sealedStateKey(tokens) + if err != nil { + return "", config.StateKey{}, err + } + log.Printf("state key: the volume is sealed in %s mode", sk.Mode) + return sk.Mode, sk, nil +} + +// sealedStateKey derives the state-key selection a volume was sealed with from +// its LUKS2 token types: a TPM token (one per bootable image after an upgrade) +// means tpm, a KMS token means kms with the endpoint and trust it carries, and +// no protector token means nodeid, which persists none. Tokens of any other +// type are ignored. +func sealedStateKey(tokens map[int][]byte) (config.StateKey, error) { + ids := make([]int, 0, len(tokens)) + for id := range tokens { + ids = append(ids, id) + } + slices.Sort(ids) + + var tpmTokens int + var kmsTok *kmsToken + for _, id := range ids { + switch tokenTypeOf(tokens[id]) { + case luks.TPM2TokenType: + tpmTokens++ + case kmsTokenType: + if kmsTok != nil { + continue + } + var t kmsToken + if err := json.Unmarshal(tokens[id], &t); err != nil { + return config.StateKey{}, fmt.Errorf("init: state volume token %d: %w", id, err) + } + kmsTok = &t + } + } + switch { + case tpmTokens > 0 && kmsTok != nil: + return config.StateKey{}, errors.New("init: the state volume has both TPM and KMS tokens; cannot tell how it was sealed") + case tpmTokens > 0: + return config.StateKey{Mode: config.StateKeyModeTPM}, nil + case kmsTok != nil: + return config.StateKey{Mode: config.StateKeyModeKMS, KMS: &config.KmsStateKey{ + Endpoint: kmsTok.Endpoint, TrustPEM: kmsTok.TrustPEM, + }}, nil + } + return config.StateKey{Mode: config.StateKeyModeNodeID}, nil +} diff --git a/internal/init/stateunlock_test.go b/internal/init/stateunlock_test.go new file mode 100644 index 0000000..732c617 --- /dev/null +++ b/internal/init/stateunlock_test.go @@ -0,0 +1,377 @@ +package init + +/* +Apache License 2.0 + +Copyright 2026 Shane + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "testing" + + cryptosv1 "github.com/CryptOS-PKI/api/go/cryptos/v1" + "github.com/CryptOS-PKI/cryptos/internal/ceremony" + "github.com/CryptOS-PKI/cryptos/internal/config" + "github.com/CryptOS-PKI/cryptos/internal/kms" + "github.com/CryptOS-PKI/cryptos/internal/storage/luks" +) + +const testKMSEndpoint = "https://kms.example" + +// headerRunner is an in-memory LUKS2 device: it answers the cryptsetup +// subcommands the state-volume code issues, keeping the key and the token set +// the way a real header would. +type headerRunner struct { + formatted bool + key []byte + tokens map[int][]byte +} + +func (r *headerRunner) Run(_ context.Context, stdin io.Reader, args ...string) ([]byte, []byte, error) { + var in []byte + if stdin != nil { + in, _ = io.ReadAll(stdin) + } + tokenID := func() int { + for i, a := range args { + if a == "--token-id" && i+1 < len(args) { + id, _ := strconv.Atoi(args[i+1]) + return id + } + } + return -1 + } + switch { + case args[0] == "isLuks": + if !r.formatted { + return nil, nil, errors.New("not a LUKS device") + } + return nil, nil, nil + case args[0] == "luksFormat": + r.formatted, r.key, r.tokens = true, in, map[int][]byte{} + return nil, nil, nil + case args[0] == "luksOpen": + if !r.formatted || !bytes.Equal(in, r.key) { + return nil, []byte("No key available with this passphrase."), errors.New("exit status 2") + } + return nil, nil, nil + case args[0] == "luksDump": + meta := map[string]map[string]json.RawMessage{"tokens": {}} + for id, tok := range r.tokens { + meta["tokens"][strconv.Itoa(id)] = tok + } + out, _ := json.Marshal(meta) + return out, nil, nil + case args[0] == "token" && args[1] == "import": + var t struct { + Type string `json:"type"` + Keyslots []string `json:"keyslots"` + } + if json.Unmarshal(in, &t) != nil || t.Type == "" || t.Keyslots == nil { + return nil, []byte("Failed to import token from file."), errors.New("exit status 1") + } + r.tokens[tokenID()] = in + return nil, nil, nil + case args[0] == "token" && args[1] == "export": + tok, ok := r.tokens[tokenID()] + if !ok { + return nil, []byte("Token is not in use."), errors.New("exit status 1") + } + return tok, nil, nil + } + return nil, nil, fmt.Errorf("headerRunner: unexpected cryptsetup %v", args) +} + +// testRunner rewrites the production cryptsetup calls so a real LUKS2 header +// in a plain file can stand in for the state partition without root: Format +// uses a cheap PBKDF, and Open only tests the key instead of mapping a device. +type testRunner struct{ exec *luks.ExecRunner } + +func (r testRunner) Run(ctx context.Context, stdin io.Reader, args ...string) ([]byte, []byte, error) { + switch args[0] { + case "luksFormat": + out := []string{"luksFormat", "--type", "luks2", "--pbkdf", "pbkdf2", "--pbkdf-force-iterations", "1000", + "--batch-mode", "--key-file", "-", args[len(args)-1]} + return r.exec.Run(ctx, stdin, out...) + case "luksOpen": + return r.exec.Run(ctx, stdin, "luksOpen", "--test-passphrase", "--key-file", "-", args[len(args)-2]) + } + return r.exec.Run(ctx, stdin, args...) +} + +// memSealer is a TPM stand-in that seals by remembering: the private blob is a +// handle for the sealed data. +type memSealer struct{ sealed map[string][]byte } + +func (m *memSealer) ProvisionSRK() error { return nil } + +func (m *memSealer) SealToPCR(data []byte, _ []int) ([]byte, []byte, error) { + if m.sealed == nil { + m.sealed = map[string][]byte{} + } + id := []byte(fmt.Sprintf("sealed-%d", len(m.sealed))) + priv := append([]byte{0, byte(len(id))}, id...) + m.sealed[string(priv)] = append([]byte(nil), data...) + return priv, []byte("public"), nil +} + +func (m *memSealer) UnsealWithPCR(priv, _ []byte, _ []int) ([]byte, error) { + data, ok := m.sealed[string(priv)] + if !ok { + return nil, errors.New("memSealer: unknown blob") + } + return append([]byte(nil), data...), nil +} + +// testBackends builds each mode's protector over fakes: the node UUID, the TPM +// and the KMS. It records the mode and state-key section each boot asked for. +type testBackends struct { + sealer memSealer + modes []string + keys []config.StateKey +} + +func (b *testBackends) factory(mode string, sk config.StateKey) (StateKeyProtector, ceremony.RootKeyBackend, func(), cryptosv1.TpmState, error) { + b.modes = append(b.modes, mode) + b.keys = append(b.keys, sk) + switch mode { + case config.StateKeyModeNodeID: + return newNodeIDProtector(fixedUUID("4c4c4544-0000-1000-8000-000000000001"), StateLabel), softRootBackend{}, + func() {}, cryptosv1.TpmState_TPM_STATE_UNAVAILABLE, nil + case config.StateKeyModeKMS: + p, err := newKMSProtector(sk.KMS) + if err != nil { + return nil, nil, func() {}, cryptosv1.TpmState_TPM_STATE_UNAVAILABLE, err + } + p.newProvider = func(string, []byte) (kms.Provider, error) { return fakeProvider{pad: 0x5a}, nil } + return p, softRootBackend{}, func() {}, cryptosv1.TpmState_TPM_STATE_UNAVAILABLE, nil + } + return newTPMProtector(&b.sealer, []int{7, 11}), nil, func() {}, cryptosv1.TpmState_TPM_STATE_OK, nil +} + +func stageYAML(mode string) []byte { + switch mode { + case "": + return []byte("metadata:\n name: ca\n") + case config.StateKeyModeKMS: + return []byte("state_key:\n mode: kms\n kms:\n endpoint: " + testKMSEndpoint + "\n") + } + return []byte("state_key:\n mode: " + mode + "\n") +} + +// realStateDevice returns a device backed by a real LUKS2 header in a file, or +// skips when cryptsetup is not available. +func realStateDevice(t *testing.T) *luks.Device { + t.Helper() + bin, err := exec.LookPath("cryptsetup") + if err != nil { + t.Skip("cryptsetup not on PATH") + } + path := filepath.Join(t.TempDir(), "state.img") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("create device file: %v", err) + } + if err := os.Truncate(path, 32<<20); err != nil { + t.Fatalf("size device file: %v", err) + } + return &luks.Device{Path: path, Runner: testRunner{exec: &luks.ExecRunner{Binary: bin}}} +} + +var modeCases = []struct { + name, buildDefault, configured, want string +}{ + {"tpm image, nodeid configured", config.StateKeyModeTPM, config.StateKeyModeNodeID, config.StateKeyModeNodeID}, + {"nodeid image, tpm configured", config.StateKeyModeNodeID, config.StateKeyModeTPM, config.StateKeyModeTPM}, + {"tpm image, kms configured", config.StateKeyModeTPM, config.StateKeyModeKMS, config.StateKeyModeKMS}, + {"nodeid image, kms configured", config.StateKeyModeNodeID, config.StateKeyModeKMS, config.StateKeyModeKMS}, + {"tpm image, no mode configured", config.StateKeyModeTPM, "", config.StateKeyModeTPM}, + {"nodeid image, no mode configured", config.StateKeyModeNodeID, "", config.StateKeyModeNodeID}, +} + +// bootAndReboot runs the first boot with the configured mode on the ESP stage, +// then two later boots after the stage is consumed, and checks that every boot +// opens the volume with the configured mode. +func bootAndReboot(t *testing.T, dev *luks.Device, buildDefault, configured, want string) { + t.Helper() + ctx := context.Background() + b := &testBackends{} + boot := func(stage []byte) *unlockedState { + t.Helper() + st, err := stateUnlocker{ + device: dev, stage: espStage(stage), buildDefault: buildDefault, newBackends: b.factory, + }.unlock(ctx) + if err != nil { + t.Fatalf("boot %d: unlock: %v", len(b.modes), err) + } + st.close() + return st + } + + if st := boot(stageYAML(configured)); !st.firstBoot || st.mode != want { + t.Fatalf("first boot: firstBoot=%v mode=%q, want true %q", st.firstBoot, st.mode, want) + } + for i := 1; i <= 2; i++ { + if st := boot(nil); st.firstBoot || st.mode != want || st.protector.Name() != want { + t.Fatalf("reboot %d: firstBoot=%v mode=%q protector=%q, want false %q", i, st.firstBoot, st.mode, st.protector.Name(), want) + } + } + if want == config.StateKeyModeKMS { + for i, sk := range b.keys[1:] { + if sk.KMS == nil || sk.KMS.Endpoint != testKMSEndpoint { + t.Errorf("reboot %d: kms section %+v, want the endpoint the volume was sealed with", i+1, sk.KMS) + } + } + } +} + +func espStage(raw []byte) espStageAccessors { + return espStageAccessors{stageReader: func() ([]byte, bool, error) { return raw, raw != nil, nil }} +} + +// A node's configured state-key mode must survive every reboot after the first, +// when the ESP stage that carried it is gone. +func TestStateUnlock_ConfiguredModeSurvivesReboots(t *testing.T) { + for _, tc := range modeCases { + t.Run(tc.name, func(t *testing.T) { + bootAndReboot(t, &luks.Device{Path: "/dev/state", Runner: &headerRunner{}}, tc.buildDefault, tc.configured, tc.want) + }) + } +} + +// The same boots against a real LUKS2 header, so the token set and the key +// check come from cryptsetup itself. +func TestStateUnlock_ConfiguredModeSurvivesReboots_RealCryptsetup(t *testing.T) { + for _, tc := range modeCases { + t.Run(tc.name, func(t *testing.T) { + bootAndReboot(t, realStateDevice(t), tc.buildDefault, tc.configured, tc.want) + }) + } +} + +// A later boot that finds a stage asking for another mode still opens the +// volume with the mode it was sealed with: the mode is fixed at install. +func TestStateUnlock_LaterStageCannotChangeTheMode(t *testing.T) { + ctx := context.Background() + dev := &luks.Device{Path: "/dev/state", Runner: &headerRunner{}} + b := &testBackends{} + unlock := func(stage []byte) (*unlockedState, error) { + return stateUnlocker{device: dev, stage: espStage(stage), buildDefault: config.StateKeyModeTPM, newBackends: b.factory}.unlock(ctx) + } + if _, err := unlock(stageYAML(config.StateKeyModeNodeID)); err != nil { + t.Fatalf("first boot: %v", err) + } + st, err := unlock(stageYAML(config.StateKeyModeTPM)) + if err != nil { + t.Fatalf("later boot with a tpm stage on a nodeid volume: %v", err) + } + if st.mode != config.StateKeyModeNodeID { + t.Errorf("mode = %q, want nodeid (the sealed mode)", st.mode) + } +} + +// After an in-place upgrade a TPM volume carries one token per bootable image; +// it is still a TPM volume, and it opens with whichever copy unseals. +func TestStateUnlock_UpgradedTPMVolume_RealCryptsetup(t *testing.T) { + ctx := context.Background() + dev := realStateDevice(t) + b := &testBackends{} + unlocker := stateUnlocker{device: dev, stage: espStage(nil), buildDefault: config.StateKeyModeNodeID, newBackends: b.factory} + unlocker.stage = espStage(stageYAML(config.StateKeyModeTPM)) + st, err := unlocker.unlock(ctx) + if err != nil { + t.Fatalf("first boot: %v", err) + } + st.close() + + // Reseal a second copy for an incoming image, then make the install-time + // copy stop unsealing, as a PCR 11 change would. + install, err := dev.ExportToken(ctx, StateTokenID) + if err != nil { + t.Fatalf("export install token: %v", err) + } + tok, _ := luks.ParseTPM2Token(install) + priv, pub, _ := tok.SealedBlobs() + key, _ := b.sealer.UnsealWithPCR(priv, pub, nil) + priv2, pub2, _ := b.sealer.SealToPCR(key, []int{7, 11}) + tok2, _ := luks.BuildTPM2Token(priv2, pub2, stateKeyslot, []int{7, 11}, nil) + tok2.ImageSHA256 = "ab" + tok2JSON, _ := json.Marshal(tok2) + if err := dev.ImportToken(ctx, StateTokenID+1, tok2JSON); err != nil { + t.Fatalf("import upgrade token: %v", err) + } + delete(b.sealer.sealed, string(priv)) + + unlocker.stage = espStage(nil) + st, err = unlocker.unlock(ctx) + if err != nil { + t.Fatalf("boot after upgrade: %v", err) + } + if st.mode != config.StateKeyModeTPM || st.firstBoot { + t.Errorf("boot after upgrade: mode=%q firstBoot=%v, want tpm false", st.mode, st.firstBoot) + } + if !slices.Equal(b.modes, []string{config.StateKeyModeTPM, config.StateKeyModeTPM}) { + t.Errorf("modes = %v", b.modes) + } +} + +func TestSealedStateKey(t *testing.T) { + tpmTok := func(image string) []byte { + tok, _ := luks.BuildTPM2Token([]byte{0, 1, 'p'}, []byte("pub"), 0, []int{7, 11}, nil) + tok.ImageSHA256 = image + out, _ := json.Marshal(tok) + return out + } + kmsTok, _ := json.Marshal(kmsToken{Type: kmsTokenType, Keyslots: []string{"0"}, Endpoint: testKMSEndpoint, TrustPEM: "trust"}) + other := []byte(`{"type":"systemd-fido2","keyslots":["1"]}`) + + cases := []struct { + name string + tokens map[int][]byte + want config.StateKey + wantErr bool + }{ + {"no tokens is nodeid", nil, config.StateKey{Mode: config.StateKeyModeNodeID}, false}, + {"foreign tokens only is nodeid", map[int][]byte{3: other}, config.StateKey{Mode: config.StateKeyModeNodeID}, false}, + {"install TPM token", map[int][]byte{0: tpmTok("")}, config.StateKey{Mode: config.StateKeyModeTPM}, false}, + {"TPM token per image after an upgrade", map[int][]byte{0: tpmTok(""), 1: tpmTok("ab"), 2: other}, config.StateKey{Mode: config.StateKeyModeTPM}, false}, + {"only an upgrade TPM token after a rollback", map[int][]byte{1: tpmTok("ab")}, config.StateKey{Mode: config.StateKeyModeTPM}, false}, + {"KMS token carries its endpoint", map[int][]byte{0: kmsTok}, + config.StateKey{Mode: config.StateKeyModeKMS, KMS: &config.KmsStateKey{Endpoint: testKMSEndpoint, TrustPEM: "trust"}}, false}, + {"TPM and KMS tokens are ambiguous", map[int][]byte{0: tpmTok(""), 1: kmsTok}, config.StateKey{}, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := sealedStateKey(tc.tokens) + if (err != nil) != tc.wantErr { + t.Fatalf("sealedStateKey err = %v, want error %v", err, tc.wantErr) + } + if got.Mode != tc.want.Mode || (got.KMS == nil) != (tc.want.KMS == nil) || (got.KMS != nil && *got.KMS != *tc.want.KMS) { + t.Errorf("sealedStateKey = %+v (kms %+v), want %+v (kms %+v)", got, got.KMS, tc.want, tc.want.KMS) + } + }) + } +} diff --git a/internal/node/apply_statekey_test.go b/internal/node/apply_statekey_test.go new file mode 100644 index 0000000..fb630a8 --- /dev/null +++ b/internal/node/apply_statekey_test.go @@ -0,0 +1,76 @@ +package node + +/* +Apache License 2.0 + +Copyright 2026 Shane + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +import ( + "bytes" + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + cryptosv1 "github.com/CryptOS-PKI/api/go/cryptos/v1" + "github.com/CryptOS-PKI/cryptos/internal/config" +) + +// A running node's state volume is sealed in one mode, and init reads the mode +// from the volume on every boot. Apply refuses a config that asks for another +// mode rather than persisting a setting the node would ignore. +func TestConfigStoreApply_RefusesStateKeyModeChange(t *testing.T) { + seed, err := config.Parse([]byte(applyValidateSeed)) + if err != nil { + t.Fatalf("config.Parse(seed): %v", err) + } + cases := []struct { + mode string + want codes.Code + }{ + {config.StateKeyModeNodeID, codes.FailedPrecondition}, + {config.StateKeyModeKMS, codes.FailedPrecondition}, + {config.StateKeyModeTPM, codes.OK}, + {"", codes.OK}, + } + for _, tc := range cases { + t.Run("mode "+tc.mode, func(t *testing.T) { + fs := config.NewFileStore(t.TempDir()) + if _, err := fs.Write([]byte(applyValidateSeed)); err != nil { + t.Fatalf("FileStore.Write (seed): %v", err) + } + beforeRaw, _, _, _ := fs.Read() + + pb := seed.ToProto() + pb.StateKey = &cryptosv1.StateKey{Mode: tc.mode} + if tc.mode == config.StateKeyModeKMS { + pb.StateKey.Kms = &cryptosv1.KmsStateKey{Endpoint: "https://kms.example"} + } + _, err := NewConfigStore(fs).WithSealedStateKeyMode(config.StateKeyModeTPM).Apply(context.Background(), pb) + if got := status.Code(err); got != tc.want { + t.Fatalf("Apply code = %v, want %v (err: %v)", got, tc.want, err) + } + if tc.want != codes.OK { + afterRaw, _, _, _ := fs.Read() + if !bytes.Equal(afterRaw, beforeRaw) { + t.Error("persisted config changed after a refused Apply") + } + } + }) + } +} diff --git a/internal/node/providers.go b/internal/node/providers.go index ef5b086..61011e8 100644 --- a/internal/node/providers.go +++ b/internal/node/providers.go @@ -204,6 +204,10 @@ type ConfigStore struct { // profiles that outlive it. Nil until WithIssuer; Apply then skips the // check. issuer IssuerFunc + + // sealedStateKeyMode is the mode the node's state volume was sealed with. + // Empty until WithSealedStateKeyMode; Apply then skips the check. + sealedStateKeyMode string } // NewConfigStore returns a ConfigStore backed by fs. @@ -219,6 +223,15 @@ func (c *ConfigStore) WithIssuer(issuer IssuerFunc) *ConfigStore { return c } +// WithSealedStateKeyMode records the mode the node's state volume was sealed +// with and returns the same ConfigStore for chaining. Apply then refuses a +// config naming another state_key.mode with codes.FailedPrecondition, because +// init reads the mode from the volume and would ignore it. +func (c *ConfigStore) WithSealedStateKeyMode(mode string) *ConfigStore { + c.sealedStateKeyMode = mode + return c +} + // ErrNoConfig is returned by ConfigStore.Current before any config has been // persisted. It carries codes.FailedPrecondition, because the node is not // ready rather than broken, so gRPC handlers can tell it apart from a genuine @@ -284,6 +297,11 @@ func (c *ConfigStore) Apply(ctx context.Context, cfg *cryptosv1.MachineConfig) ( if err != nil { return nil, status.Errorf(codes.InvalidArgument, "node: Apply: %v", err) } + if c.sealedStateKeyMode != "" { + if err := parsed.StateKey.CheckSealed(c.sealedStateKeyMode); err != nil { + return nil, status.Errorf(codes.FailedPrecondition, "node: Apply: %v", err) + } + } requiresReboot := config.NeedsReboot(oldCfg, parsed) log.Printf("node: Apply: acme=%t est=%t requires_reboot=%t", parsed.PKI.ACME != nil, parsed.PKI.EST != nil, requiresReboot) From 75d63cd5a6a5cfc56351b913d280405cf866452c Mon Sep 17 00:00:00 2001 From: Bugs5382 <12115015+Bugs5382@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:40:30 -0400 Subject: [PATCH 2/3] docs: the state-key mode is fixed at install and survives upgrades Signed-off-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com> --- build/README.md | 15 +++++++++++++++ docs/image-upgrade.md | 11 ++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/build/README.md b/build/README.md index 02459c2..10722a0 100644 --- a/build/README.md +++ b/build/README.md @@ -172,6 +172,21 @@ stored on the encrypted state partition. `cryptosctl status` reports > the Root key. Use `nodeid` to run CryptOS where a vTPM is unavailable, never for > a CA guarding real trust. +`STATEKEY` is only the image's default. A machine config can pick another mode +with `state_key.mode` (`tpm`, `nodeid` or `kms`), and that choice is read from +the config staged for the first boot, when the state partition is formatted. +From then on the mode is fixed: every later boot, including one into an +upgraded image built with a different `STATEKEY`, reads it from the state +partition's LUKS2 header (a `cryptos-tpm2` token means `tpm`, a `cryptos-kms` +token means `kms`, no protector token means `nodeid`), not from the image or +the config. + +> [!CAUTION] +> The state-key mode cannot be changed on an installed node. `config apply` of a +> config naming a different `state_key.mode` is refused with +> `FailedPrecondition` and nothing is written. To change the mode, reinstall +> the node. + ## Machine config delivery The image is **config-free**: the rootfs carries no `machine.yaml`. Machine diff --git a/docs/image-upgrade.md b/docs/image-upgrade.md index 4b737ac..456bbd1 100644 --- a/docs/image-upgrade.md +++ b/docs/image-upgrade.md @@ -192,9 +192,18 @@ poor trade against PKCS#1 v1.5 over a SHA-256 digest. - **Only one previous image is retained.** Two upgrades in a row leave you able to roll back one. +## State-key mode + +An upgrade never changes how the state partition is unlocked. The node reads +its state-key mode from the partition's LUKS2 header on every boot, so the mode +chosen at install (from `state_key.mode`, or the installing image's `STATEKEY` +default) carries over to any image, whatever `STATEKEY` the new image was built +with. A `nodeid` node upgraded to a `STATEKEY=tpm` image stays `nodeid`, and +the reverse. + ## TPM-backed nodes -On a `STATEKEY=tpm` node the state-partition key is sealed to PCR 7 (Secure +On a `tpm`-mode node the state-partition key is sealed to PCR 7 (Secure Boot policy) and PCR 11. PCR 11 is where systemd-stub measures the UKI, so every new image changes it, and a key sealed only to the running image would not unseal after the upgrade. From beac95ecf59260c4a201fa49371e352acff5091d Mon Sep 17 00:00:00 2001 From: Bugs5382 <12115015+Bugs5382@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:20:23 -0400 Subject: [PATCH 3/3] style: gofmt the new state-key files Signed-off-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com> --- internal/config/statekey_sealed_test.go | 1 - internal/init/stateunlock.go | 1 - internal/init/stateunlock_test.go | 1 - internal/node/apply_statekey_test.go | 1 - 4 files changed, 4 deletions(-) diff --git a/internal/config/statekey_sealed_test.go b/internal/config/statekey_sealed_test.go index 035c454..c522f31 100644 --- a/internal/config/statekey_sealed_test.go +++ b/internal/config/statekey_sealed_test.go @@ -18,7 +18,6 @@ See the License for the specific language governing permissions and limitations under the License. */ - import ( "strings" "testing" diff --git a/internal/init/stateunlock.go b/internal/init/stateunlock.go index eae2848..a68473b 100644 --- a/internal/init/stateunlock.go +++ b/internal/init/stateunlock.go @@ -18,7 +18,6 @@ See the License for the specific language governing permissions and limitations under the License. */ - import ( "context" "encoding/json" diff --git a/internal/init/stateunlock_test.go b/internal/init/stateunlock_test.go index 732c617..02e99d8 100644 --- a/internal/init/stateunlock_test.go +++ b/internal/init/stateunlock_test.go @@ -18,7 +18,6 @@ See the License for the specific language governing permissions and limitations under the License. */ - import ( "bytes" "context" diff --git a/internal/node/apply_statekey_test.go b/internal/node/apply_statekey_test.go index fb630a8..d5e033f 100644 --- a/internal/node/apply_statekey_test.go +++ b/internal/node/apply_statekey_test.go @@ -18,7 +18,6 @@ See the License for the specific language governing permissions and limitations under the License. */ - import ( "bytes" "context"