fix(persistence): apply rqlite schema in-process via an init container

The schemahero 0.25 bump broke the schemahero-plan/apply init containers:
0.25 moved the rqlite driver into a go-plugin subprocess and 'schemahero
apply' hung, leaving kotsadm stuck in Init and failing nearly all e2e
validation jobs.

Replace those two init containers with a single rqlite-migrations init
container that runs 'kotsadm migrate rqlite-schema'. That command applies
the schema in-process through the schemahero driver lib linked into
kotsadm, from table specs embedded in the binary, so there is no plugin
binary to ship or discover. Running it as an init container keeps the
schema in place before the main container and the restore init containers,
preserving the existing ordering.

- Add 'kotsadm migrate rqlite-schema' (persistence.UpdateRqliteSchema over
  the embedded migrations.Tables specs)
- Replace schemahero-plan/apply with the rqlite-migrations init container
  in the kotsadm Deployment and StatefulSet
- Revert the migrations-image schemahero/plugin change; the image is
  unchanged from main and still serves wait-for-rqlite and the kURL
  schemahero init containers (which keep the 0.22 binary)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: kriscoleman

cmd/kotsadm/cli/migrate.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/replicatedhq/kots/pkg/filestore"
+	"github.com/replicatedhq/kots/pkg/persistence"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -21,6 +22,38 @@ func MigrateCmd() *cobra.Command {
 
 	cmd.AddCommand(MigrateS3ToRqliteCmd())
 	cmd.AddCommand(MigratePVCToRqliteCmd())
+	cmd.AddCommand(MigrateRqliteSchemaCmd())
+
+	return cmd
+}
+
+// MigrateRqliteSchemaCmd applies the rqlite schema in-process using the table
+// specs embedded in the kotsadm binary. It runs as an init container in place of
+// the schemahero-plan/apply containers, so the schema is created by the
+// linked-in schemahero driver lib with no plugin binary to ship or discover.
+func MigrateRqliteSchemaCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:           "rqlite-schema",
+		Short:         "Apply the rqlite schema in-process",
+		Long:          ``,
+		SilenceUsage:  true,
+		SilenceErrors: false,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			viper.BindPFlags(cmd.Flags())
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			uri := os.Getenv("RQLITE_URI")
+			if uri == "" {
+				return errors.New("RQLITE_URI is not set")
+			}
+
+			if err := persistence.UpdateRqliteSchema(uri); err != nil {
+				return errors.Wrap(err, "failed to update rqlite schema")
+			}
+
+			return nil
+		},
+	}
 
 	return cmd
 }

migrations/embed.go

@@ -0,0 +1,11 @@
+package migrations
+
+import "embed"
+
+// Tables holds the schemahero table specs used to create and update the rqlite
+// schema in-process at kotsadm startup (see persistence.UpdateRqliteSchema).
+// Embedding the specs in the kotsadm binary means the migration needs no
+// schemahero plugin binary and nothing to ship or discover at runtime.
+//
+//go:embed tables
+var Tables embed.FS

pkg/kotsadm/objects/kotsadm_objects.go

@@ -201,8 +201,9 @@ func updateKotsadmDeploymentScriptsPath(existing *appsv1.Deployment) {
 }
 
 // waitForRqliteInitContainer returns an init container that polls the rqlite
-// readiness endpoint before schemahero-plan runs. This prevents CrashLoopBackOff
-// when kotsadm and rqlite restart simultaneously (e.g., during EC upgrades).
+// readiness endpoint before the rqlite-migrations init container runs. This
+// prevents CrashLoopBackOff when kotsadm and rqlite restart simultaneously
+// (e.g., during EC upgrades).
 // Times out after 5 minutes so rqlite failures surface as a clear init error
 // rather than an indefinite hang.
 // Ref: https://app.shortcut.com/replicated/story/138103
@@ -226,6 +227,44 @@ func waitForRqliteInitContainer(deployOptions types.DeployOptions) corev1.Contai
 	}
 }
 
+// rqliteMigrationsInitContainer returns an init container that applies the
+// rqlite schema in-process by running the kotsadm binary's `migrate
+// rqlite-schema` command. It replaces the schemahero-plan/apply init containers:
+// the schema is created by the schemahero driver lib linked into kotsadm (no
+// plugin binary to ship or discover), and running it as an init container keeps
+// the schema in place before the main container and the restore init containers.
+func rqliteMigrationsInitContainer(deployOptions types.DeployOptions) corev1.Container {
+	return corev1.Container{
+		Image:           GetAdminConsoleImage(deployOptions, "kotsadm"),
+		ImagePullPolicy: corev1.PullIfNotPresent,
+		Name:            "rqlite-migrations",
+		Command:         []string{"/kotsadm", "migrate", "rqlite-schema"},
+		Env: []corev1.EnvVar{
+			{
+				Name: "RQLITE_URI",
+				ValueFrom: &corev1.EnvVarSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: "kotsadm-rqlite",
+						},
+						Key: "uri",
+					},
+				},
+			},
+		},
+		Resources: corev1.ResourceRequirements{
+			Limits: corev1.ResourceList{
+				"memory": resource.MustParse("100Mi"),
+			},
+			Requests: corev1.ResourceList{
+				"cpu":    resource.MustParse("50m"),
+				"memory": resource.MustParse("50Mi"),
+			},
+		},
+		SecurityContext: k8sutil.SecureContainerContext(deployOptions.StrictSecurityContext),
+	}
+}
+
 func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, error) {
 	securityContext := k8sutil.SecurePodContext(1001, 1001, deployOptions.StrictSecurityContext)
 	if deployOptions.IsOpenShift {
@@ -438,14 +477,6 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e
 	}
 
 	volumes := []corev1.Volume{
-		{
-			Name: "migrations",
-			VolumeSource: corev1.VolumeSource{
-				EmptyDir: &corev1.EmptyDirVolumeSource{
-					Medium: corev1.StorageMediumMemory,
-				},
-			},
-		},
 		{
 			Name: "backup",
 			VolumeSource: corev1.VolumeSource{
@@ -525,112 +556,7 @@ func KotsadmDeployment(deployOptions types.DeployOptions) (*appsv1.Deployment, e
 					ImagePullSecrets:   pullSecrets,
 					InitContainers: []corev1.Container{
 						waitForRqliteInitContainer(deployOptions),
-						{
-							Image:           GetAdminConsoleImage(deployOptions, "kotsadm-migrations"),
-							ImagePullPolicy: corev1.PullIfNotPresent,
-							Name:            "schemahero-plan",
-							Args:            []string{"plan"},
-							VolumeMounts: []corev1.VolumeMount{
-								{
-									Name:      "migrations",
-									MountPath: "/migrations",
-								},
-								{
-									Name:      "tmp",
-									MountPath: "/tmp",
-								},
-							},
-							Env: []corev1.EnvVar{
-								{
-									Name:  "HOME",
-									Value: "/tmp",
-								},
-								{
-									Name:  "SCHEMAHERO_DRIVER",
-									Value: "rqlite",
-								},
-								{
-									Name:  "SCHEMAHERO_SPEC_FILE",
-									Value: "/tables",
-								},
-								{
-									Name:  "SCHEMAHERO_OUT",
-									Value: "/migrations/plan.yaml",
-								},
-								{
-									Name: "SCHEMAHERO_URI",
-									ValueFrom: &corev1.EnvVarSource{
-										SecretKeyRef: &corev1.SecretKeySelector{
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: "kotsadm-rqlite",
-											},
-											Key: "uri",
-										},
-									},
-								},
-							},
-							Resources: corev1.ResourceRequirements{
-								Limits: corev1.ResourceList{
-									"memory": resource.MustParse("100Mi"),
-								},
-								Requests: corev1.ResourceList{
-									"cpu":    resource.MustParse("50m"),
-									"memory": resource.MustParse("50Mi"),
-								},
-							},
-							SecurityContext: k8sutil.SecureContainerContext(deployOptions.StrictSecurityContext),
-						},
-						{
-							Image:           GetAdminConsoleImage(deployOptions, "kotsadm-migrations"),
-							ImagePullPolicy: corev1.PullIfNotPresent,
-							Name:            "schemahero-apply",
-							Args:            []string{"apply"},
-							VolumeMounts: []corev1.VolumeMount{
-								{
-									Name:      "migrations",
-									MountPath: "/migrations",
-								},
-								{
-									Name:      "tmp",
-									MountPath: "/tmp",
-								},
-							},
-							Env: []corev1.EnvVar{
-								{
-									Name:  "HOME",
-									Value: "/tmp",
-								},
-								{
-									Name:  "SCHEMAHERO_DRIVER",
-									Value: "rqlite",
-								},
-								{
-									Name:  "SCHEMAHERO_DDL",
-									Value: "/migrations/plan.yaml",
-								},
-								{
-									Name: "SCHEMAHERO_URI",
-									ValueFrom: &corev1.EnvVarSource{
-										SecretKeyRef: &corev1.SecretKeySelector{
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: "kotsadm-rqlite",
-											},
-											Key: "uri",
-										},
-									},
-								},
-							},
-							Resources: corev1.ResourceRequirements{
-								Limits: corev1.ResourceList{
-									"memory": resource.MustParse("100Mi"),
-								},
-								Requests: corev1.ResourceList{
-									"cpu":    resource.MustParse("50m"),
-									"memory": resource.MustParse("50Mi"),
-								},
-							},
-							SecurityContext: k8sutil.SecureContainerContext(deployOptions.StrictSecurityContext),
-						},
+						rqliteMigrationsInitContainer(deployOptions),
 						{
 							Image:           GetAdminConsoleImage(deployOptions, "kotsadm"),
 							ImagePullPolicy: corev1.PullIfNotPresent,
@@ -1027,14 +953,6 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit
 				},
 			},
 		},
-		{
-			Name: "migrations",
-			VolumeSource: corev1.VolumeSource{
-				EmptyDir: &corev1.EmptyDirVolumeSource{
-					Medium: corev1.StorageMediumMemory,
-				},
-			},
-		},
 		{
 			Name: "backup",
 			VolumeSource: corev1.VolumeSource{
@@ -1119,112 +1037,7 @@ func KotsadmStatefulSet(deployOptions types.DeployOptions, size resource.Quantit
 					ImagePullSecrets:   pullSecrets,
 					InitContainers: []corev1.Container{
 						waitForRqliteInitContainer(deployOptions),
-						{
-							Image:           GetAdminConsoleImage(deployOptions, "kotsadm-migrations"),
-							ImagePullPolicy: corev1.PullIfNotPresent,
-							Name:            "schemahero-plan",
-							Args:            []string{"plan"},
-							VolumeMounts: []corev1.VolumeMount{
-								{
-									Name:      "migrations",
-									MountPath: "/migrations",
-								},
-								{
-									Name:      "tmp",
-									MountPath: "/tmp",
-								},
-							},
-							Env: []corev1.EnvVar{
-								{
-									Name:  "HOME",
-									Value: "/tmp",
-								},
-								{
-									Name:  "SCHEMAHERO_DRIVER",
-									Value: "rqlite",
-								},
-								{
-									Name:  "SCHEMAHERO_SPEC_FILE",
-									Value: "/tables",
-								},
-								{
-									Name:  "SCHEMAHERO_OUT",
-									Value: "/migrations/plan.yaml",
-								},
-								{
-									Name: "SCHEMAHERO_URI",
-									ValueFrom: &corev1.EnvVarSource{
-										SecretKeyRef: &corev1.SecretKeySelector{
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: "kotsadm-rqlite",
-											},
-											Key: "uri",
-										},
-									},
-								},
-							},
-							Resources: corev1.ResourceRequirements{
-								Limits: corev1.ResourceList{
-									"memory": resource.MustParse("100Mi"),
-								},
-								Requests: corev1.ResourceList{
-									"cpu":    resource.MustParse("50m"),
-									"memory": resource.MustParse("50Mi"),
-								},
-							},
-							SecurityContext: k8sutil.SecureContainerContext(deployOptions.StrictSecurityContext),
-						},
-						{
-							Image:           GetAdminConsoleImage(deployOptions, "kotsadm-migrations"),
-							ImagePullPolicy: corev1.PullIfNotPresent,
-							Name:            "schemahero-apply",
-							Args:            []string{"apply"},
-							VolumeMounts: []corev1.VolumeMount{
-								{
-									Name:      "migrations",
-									MountPath: "/migrations",
-								},
-								{
-									Name:      "tmp",
-									MountPath: "/tmp",
-								},
-							},
-							Env: []corev1.EnvVar{
-								{
-									Name:  "HOME",
-									Value: "/tmp",
-								},
-								{
-									Name:  "SCHEMAHERO_DRIVER",
-									Value: "rqlite",
-								},
-								{
-									Name:  "SCHEMAHERO_DDL",
-									Value: "/migrations/plan.yaml",
-								},
-								{
-									Name: "SCHEMAHERO_URI",
-									ValueFrom: &corev1.EnvVarSource{
-										SecretKeyRef: &corev1.SecretKeySelector{
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: "kotsadm-rqlite",
-											},
-											Key: "uri",
-										},
-									},
-								},
-							},
-							Resources: corev1.ResourceRequirements{
-								Limits: corev1.ResourceList{
-									"memory": resource.MustParse("100Mi"),
-								},
-								Requests: corev1.ResourceList{
-									"cpu":    resource.MustParse("50m"),
-									"memory": resource.MustParse("50Mi"),
-								},
-							},
-							SecurityContext: k8sutil.SecureContainerContext(deployOptions.StrictSecurityContext),
-						},
+						rqliteMigrationsInitContainer(deployOptions),
 						{
 							Image:           GetAdminConsoleImage(deployOptions, "kotsadm"),
 							ImagePullPolicy: corev1.PullIfNotPresent,

pkg/kotsadm/objects/kotsadm_objects_test.go

@@ -266,7 +266,13 @@ func Test_kotsadmDeploymentHasWaitForRqlite(t *testing.T) {
 	require.NoError(t, err)
 
 	initContainers := dep.Spec.Template.Spec.InitContainers
-	require.True(t, len(initContainers) >= 4, "expected at least 4 init containers, got %d", len(initContainers))
+	require.GreaterOrEqual(t, len(initContainers), 2)
 	assert.Equal(t, "wait-for-rqlite", initContainers[0].Name, "wait-for-rqlite should be the first init container")
-	assert.Equal(t, "schemahero-plan", initContainers[1].Name, "schemahero-plan should follow wait-for-rqlite")
+	assert.Equal(t, "rqlite-migrations", initContainers[1].Name, "rqlite-migrations should run right after wait-for-rqlite")
+
+	// The rqlite schema is applied in-process by the rqlite-migrations init
+	// container, so there should be no schemahero migration init containers.
+	for _, c := range initContainers {
+		assert.NotContains(t, c.Name, "schemahero", "schemahero init containers should be replaced by the in-process rqlite-migrations container")
+	}
 }

pkg/kotsadm/objects/scripts/wait-for-rqlite.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Polls the rqlite readiness endpoint before schemahero-plan runs.
+# Polls the rqlite readiness endpoint before the rqlite-migrations init container runs.
 # Prevents CrashLoopBackOff when kotsadm and rqlite restart simultaneously
 # (e.g., during Embedded Cluster upgrades).
 # Times out after 5 minutes so rqlite failures surface as a clear init error