fix(init): read the state-key mode from the volume after the first boot (#305)

* fix(init): read the state-key mode from the volume after the first boot

A node whose state_key.mode came from its machine config and differed
from the image's build-time default could not come back from any reboot.
init resolved the mode before unlock from the ESP config stage only, and
the stage exists only on the first boot, so every later boot fell back to
the build-time mode and opened the volume with the wrong protector.

Later boots now derive the mode from the LUKS2 header the volume was
sealed with: a cryptos-tpm2 token means tpm, a cryptos-kms token means
kms (with the endpoint and trust it carries), and no protector token
means nodeid. The first boot still takes it from the staged config, else
the build-time default. The multi-token TPM header left by an upgrade
still reads as tpm.

The KMS token carried no type or keyslots, which LUKS2 requires, so
cryptsetup refused to import it and kms mode could not finish a first
boot. It now has type cryptos-kms and the state keyslot.

The mode is fixed at install. ApplyConfig on a running node and the
re-provision installer refuse a config naming another state_key.mode with
FailedPrecondition, and a boot whose persisted config disagrees logs it
and keeps the sealed mode.

Signed-off-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com>

* docs: the state-key mode is fixed at install and survives upgrades

Signed-off-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com>

* style: gofmt the new state-key files

Signed-off-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com>

---------

Signed-off-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com>
Co-authored-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com>

Author: Bugs5382

build/README.md

@@ -172,6 +172,21 @@ stored on the encrypted state partition. `cryptosctl status` reports
 > the Root key. Use `nodeid` to run CryptOS where a vTPM is unavailable, never for
 > a CA guarding real trust.
 
+`STATEKEY` is only the image's default. A machine config can pick another mode
+with `state_key.mode` (`tpm`, `nodeid` or `kms`), and that choice is read from
+the config staged for the first boot, when the state partition is formatted.
+From then on the mode is fixed: every later boot, including one into an
+upgraded image built with a different `STATEKEY`, reads it from the state
+partition's LUKS2 header (a `cryptos-tpm2` token means `tpm`, a `cryptos-kms`
+token means `kms`, no protector token means `nodeid`), not from the image or
+the config.
+
+> [!CAUTION]
+> The state-key mode cannot be changed on an installed node. `config apply` of a
+> config naming a different `state_key.mode` is refused with
+> `FailedPrecondition` and nothing is written. To change the mode, reinstall
+> the node.
+
 ## Machine config delivery
 
 The image is **config-free**: the rootfs carries no `machine.yaml`. Machine

docs/image-upgrade.md

@@ -192,9 +192,18 @@ poor trade against PKCS#1 v1.5 over a SHA-256 digest.
 - **Only one previous image is retained.** Two upgrades in a row leave you able
   to roll back one.
 
+## State-key mode
+
+An upgrade never changes how the state partition is unlocked. The node reads
+its state-key mode from the partition's LUKS2 header on every boot, so the mode
+chosen at install (from `state_key.mode`, or the installing image's `STATEKEY`
+default) carries over to any image, whatever `STATEKEY` the new image was built
+with. A `nodeid` node upgraded to a `STATEKEY=tpm` image stays `nodeid`, and
+the reverse.
+
 ## TPM-backed nodes
 
-On a `STATEKEY=tpm` node the state-partition key is sealed to PCR 7 (Secure
+On a `tpm`-mode node the state-partition key is sealed to PCR 7 (Secure
 Boot policy) and PCR 11. PCR 11 is where systemd-stub measures the UKI, so
 every new image changes it, and a key sealed only to the running image would
 not unseal after the upgrade.

internal/config/config.go

@@ -144,12 +144,24 @@ type Config struct {
 // StateKey selects the protector for the encrypted state-partition key. Mode is
 // "" (build-time default) | "nodeid" | "tpm" | "kms". It is carried in the proto
 // MachineConfig so the choice survives ApplyConfig and reaches an installed node
-// (the maintenance installer reconstructs the staged YAML from the proto).
+// (the maintenance installer reconstructs the staged YAML from the proto). The
+// mode takes effect when the state volume is first sealed; every later boot
+// reads it from the volume.
 type StateKey struct {
 	Mode string       `yaml:"mode"`
 	KMS  *KmsStateKey `yaml:"kms"`
 }
 
+// CheckSealed refuses a mode other than sealed, the mode the node's state
+// volume was sealed with at install. An empty mode asks for nothing and passes.
+func (sk StateKey) CheckSealed(sealed string) error {
+	if sk.Mode == "" || sk.Mode == sealed {
+		return nil
+	}
+	return fmt.Errorf("config: state_key.mode %q: this node's state volume is sealed in %q mode, which is fixed at install; reinstall the node to change it",
+		sk.Mode, sealed)
+}
+
 // KmsStateKey configures the envelope-encryption KMS that seals/unseals the
 // state key. Only first-boot provisioning reads this from the machine config;
 // later boots recover the endpoint and sealed blob from the LUKS2 header token.

internal/config/statekey_sealed_test.go

@@ -0,0 +1,50 @@
+package config
+
+/*
+Apache License 2.0
+
+Copyright 2026 Shane
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import (
+	"strings"
+	"testing"
+)
+
+// The state-key mode is fixed when the volume is sealed at install. A config
+// may repeat it or leave it empty, but may not name another mode.
+func TestStateKeyCheckSealed(t *testing.T) {
+	cases := []struct {
+		name, configured, sealed string
+		wantErr                  bool
+	}{
+		{"empty keeps the sealed mode", "", StateKeyModeTPM, false},
+		{"same mode", StateKeyModeNodeID, StateKeyModeNodeID, false},
+		{"tpm on a nodeid volume", StateKeyModeTPM, StateKeyModeNodeID, true},
+		{"nodeid on a tpm volume", StateKeyModeNodeID, StateKeyModeTPM, true},
+		{"kms on a tpm volume", StateKeyModeKMS, StateKeyModeTPM, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := StateKey{Mode: tc.configured}.CheckSealed(tc.sealed)
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("CheckSealed = %v, want error %v", err, tc.wantErr)
+			}
+			if err != nil && (!strings.Contains(err.Error(), tc.configured) || !strings.Contains(err.Error(), tc.sealed)) {
+				t.Errorf("error %q should name both modes", err)
+			}
+		})
+	}
+}

internal/init/configload.go

@@ -100,12 +100,11 @@ func loadOrSeedConfig(store *config.FileStore, stage espStageAccessors) (*config
 	return nil, fmt.Errorf("%w: no persisted config on an installed node", errEnterMaintenance)
 }
 
-// preUnlockStateKey resolves the state-key selection readable before the
-// encrypted volume is open: the ESP stage (present on first boot with the
-// operator's config). It parses only the state_key section, tolerating an
-// absent or unparseable stage by returning the zero StateKey — the caller then
-// falls back to the build-time default and the volume opens with the token on a
-// later boot. It never fails the boot: pre-unlock config is best-effort.
+// preUnlockStateKey reads the first boot's state-key selection from the ESP
+// stage, which carries the operator's config before the volume exists. It
+// parses only the state_key section, tolerating an absent or unparseable stage
+// by returning the zero StateKey, so the caller falls back to the build-time
+// default. Later boots never call it: they read the mode from the volume.
 func preUnlockStateKey(stage espStageAccessors) config.StateKey {
 	if stage.stageReader == nil {
 		return config.StateKey{}

internal/init/kms_protector.go

@@ -24,6 +24,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strconv"
 
 	"github.com/CryptOS-PKI/cryptos/internal/config"
 	"github.com/CryptOS-PKI/cryptos/internal/kms"
@@ -42,13 +43,19 @@ type kmsProtector struct {
 	newProvider func(endpoint string, trustPEM []byte) (kms.Provider, error)
 }
 
+// kmsTokenType is the LUKS2 token type of the KMS protector's token. The type
+// is also how a later boot knows the volume was sealed in kms mode.
+const kmsTokenType = "cryptos-kms"
+
 // kmsToken is the JSON persisted in the LUKS2 header token. It carries the
 // endpoint and trust bundle so later boots can rebuild the Provider without the
-// machine config, plus the sealed DEK blob.
+// machine config, plus the sealed DEK blob. LUKS2 requires type and keyslots.
 type kmsToken struct {
-	Endpoint string `json:"endpoint"`
-	TrustPEM string `json:"trust_pem"`
-	Sealed   []byte `json:"sealed"`
+	Type     string   `json:"type"`
+	Keyslots []string `json:"keyslots"`
+	Endpoint string   `json:"endpoint"`
+	TrustPEM string   `json:"trust_pem"`
+	Sealed   []byte   `json:"sealed"`
 }
 
 // newKMSProtector builds a kmsProtector from the machine-config KMS settings.
@@ -99,6 +106,8 @@ func (p *kmsProtector) ProvisionKey(ctx context.Context) (key, token []byte, err
 		return nil, nil, fmt.Errorf("kms: seal: %w", err)
 	}
 	token, err = json.Marshal(kmsToken{
+		Type:     kmsTokenType,
+		Keyslots: []string{strconv.Itoa(stateKeyslot)},
 		Endpoint: p.endpoint,
 		TrustPEM: string(p.trustPEM),
 		Sealed:   sealed,

internal/init/maintenance.go

@@ -140,7 +140,7 @@ func runMaintenance(ctx context.Context) error {
 // The listener setup mirrors runMaintenance: self-signed cert, client auth OFF
 // (Talos --insecure), no-op auditor. It parks until a shutdown signal or a
 // successful re-provision (which triggers reboot), then returns so PID 1 reboots.
-func runReprovisionMaintenance(ctx context.Context, cfgStore *config.FileStore) error {
+func runReprovisionMaintenance(ctx context.Context, cfgStore *config.FileStore, sealedMode string) error {
 	if err := netlink.BringUpLoopback(); err != nil {
 		return err
 	}
@@ -154,7 +154,7 @@ func runReprovisionMaintenance(ctx context.Context, cfgStore *config.FileStore)
 	// client before tearing down the connection, then this function returns and
 	// PID 1 reboots into the ceremony.
 	rebootCh := make(chan struct{}, 1)
-	rp := &reprovisioner{store: cfgStore, rebootCh: rebootCh}
+	rp := &reprovisioner{store: cfgStore, rebootCh: rebootCh, sealedMode: sealedMode}
 
 	srv, err := cgrpc.NewMaintenance(cgrpc.ServerConfig{
 		TLSConfig:  MaintenanceServerTLSConfig(serverCert),

internal/init/reprovision.go

@@ -50,6 +50,8 @@ import (
 type reprovisioner struct {
 	store    *config.FileStore
 	rebootCh chan struct{}
+	// sealedMode is the state-key mode the kept state volume was sealed with.
+	sealedMode string
 }
 
 // Install validates cfg, persists it to the mounted state store, signals a
@@ -60,6 +62,9 @@ func (r *reprovisioner) Install(_ context.Context, cfg *cryptosv1.MachineConfig)
 	if err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "reprovision: parse: %v", err)
 	}
+	if err := parsed.StateKey.CheckSealed(r.sealedMode); err != nil {
+		return nil, status.Errorf(codes.FailedPrecondition, "reprovision: %v", err)
+	}
 	if err := parsed.Validate(); err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "reprovision: validate: %v", err)
 	}

internal/init/reprovision_test.go

@@ -25,6 +25,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	cryptosv1 "github.com/CryptOS-PKI/api/go/cryptos/v1"
 	"github.com/CryptOS-PKI/cryptos/internal/config"
 )
 
@@ -114,3 +115,27 @@ func TestReprovisioner_InvalidConfig(t *testing.T) {
 		// expected: no reboot signal
 	}
 }
+
+// Re-provisioning keeps the state volume, so it keeps the mode the volume was
+// sealed with. A config naming another mode is refused, not persisted, and
+// does not reboot.
+func TestReprovisioner_RefusesStateKeyModeChange(t *testing.T) {
+	store := config.NewFileStore(t.TempDir())
+	rebootCh := make(chan struct{}, 1)
+	rp := &reprovisioner{store: store, rebootCh: rebootCh, sealedMode: config.StateKeyModeNodeID}
+
+	cfg := validMachineConfig()
+	cfg.StateKey = &cryptosv1.StateKey{Mode: config.StateKeyModeTPM}
+	_, err := rp.Install(context.Background(), cfg)
+	if got := status.Code(err); got != codes.FailedPrecondition {
+		t.Fatalf("Install code = %v, want FailedPrecondition (err: %v)", got, err)
+	}
+	if _, _, ok, _ := store.Read(); ok {
+		t.Error("config persisted after a refused re-provision")
+	}
+	select {
+	case <-rebootCh:
+		t.Error("reboot signalled after a refused re-provision")
+	default:
+	}
+}

internal/init/run.go

@@ -210,45 +210,30 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) {
 	}
 	begin("state volume")
 
-	// 3. State-key + Root-key backends (TPM-sealed by default; nodeID/software
-	// or KMS-wrapped for the TPM-less variants). The state-key selection is
-	// resolved pre-unlock: on first boot the ESP-staged config (readable before
-	// the volume opens) supplies the kms endpoint for ProvisionKey; on later
-	// boots the staged config is gone and the build-time StateKeyMode default
-	// applies (RecoverKey then reads the endpoint from the LUKS2 token, not
-	// config). The effective mode is the staged config's when set, else the
-	// build-time default.
-	sk := preUnlockStateKey(realESPStageAccessors())
-	mode := StateKeyMode
-	if sk.Mode != "" {
-		mode = sk.Mode
-	}
-	protector, rootBackend, closeBackends, tpmState, err := newStateKeyBackends(mode, sk)
-	if err != nil {
-		return err
-	}
-	defer closeBackends()
-	log.Printf("state key mode: %s", protector.Name())
-
-	// 4. Open (or first-boot-format) the encrypted state volume. Resolve the
-	// state partition by its GPT name via sysfs (the image has no udev, so the
-	// by-partlabel symlinks never exist); devtmpfs has created the /dev node.
-	// First-boot is decided from the partition itself (!IsLUKS), not from config,
-	// because config does not exist yet.
+	// 3. Resolve the state partition by its GPT name via sysfs (the image has no
+	// udev, so the by-partlabel symlinks never exist); devtmpfs has created the
+	// /dev node. First-boot is decided from the partition itself (!IsLUKS), not
+	// from config, because config does not exist yet.
 	stateDevice, err := resolveStateDevice(StateLabel)
 	if err != nil {
 		return err
 	}
 	paths.Device = stateDevice
 	dev := &luks.Device{Path: paths.Device, Runner: &luks.ExecRunner{Binary: cryptsetupBinary}}
-	firstBoot := !dev.IsLUKS(ctx)
-	vol, err := OpenStateVolume(ctx, StateVolumeConfig{
-		Protector: protector, Device: dev, MappedName: StateMappedName,
-		TokenID: StateTokenID, FirstBoot: firstBoot,
-	})
+
+	// 4. State-key + Root-key backends (TPM-sealed by default; nodeID/software
+	// or KMS-wrapped for the TPM-less variants), then open (or first-boot-format)
+	// the encrypted state volume with them.
+	st, err := stateUnlocker{
+		device: dev, stage: realESPStageAccessors(),
+		buildDefault: StateKeyMode, newBackends: newStateKeyBackends,
+	}.unlock(ctx)
 	if err != nil {
 		return err
 	}
+	defer st.close()
+	log.Printf("state key mode: %s", st.protector.Name())
+	mode, rootBackend, tpmState, vol, firstBoot := st.mode, st.root, st.tpmState, st.vol, st.firstBoot
 	if firstBoot {
 		if err := mkfsExt4(vol.Path); err != nil {
 			return err
@@ -292,10 +277,13 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) {
 			// which persists the applied config to the mounted state and reboots
 			// into the ceremony, rather than the bare-disk installer.
 			log.Printf("REPROVISION: %v", err)
-			return runReprovisionMaintenance(ctx, cfgStore)
+			return runReprovisionMaintenance(ctx, cfgStore, mode)
 		}
 		return err
 	}
+	if err := cfg.StateKey.CheckSealed(mode); err != nil {
+		log.Printf("state key: %v; using the sealed mode", err)
+	}
 	done()
 	begin("network")
 
@@ -441,7 +429,7 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) {
 			Identity:    node.NewIdentityProvider(store),
 			Status:      statusProv,
 			Ceremony:    eng,
-			ConfigStore: node.NewConfigStore(cfgStore).WithIssuer(issuerFunc),
+			ConfigStore: node.NewConfigStore(cfgStore).WithIssuer(issuerFunc).WithSealedStateKeyMode(mode),
 		}
 	}