Use Kubernetes Secrets for backend bootstrap authentication (#1275)

* Use Kubernetes Secrets for backend bootstrap authentication

* Add safe Helm bootstrap for backend tokens

* Fix backend secret auth test cleanup

* Prevent backend token disclosure in deployment helpers

* Resolve MCP and backend token chart integration

* Address backend bootstrap review feedback

* Replace backend bootstrap service image with kubectl hook

* Run backend bootstrap with BusyBox ash

* Fix optional token lookup and cache publication

* Install kubectl in internal CI

Author: vvnpn-nv

.github/workflows/helm-chart-lint.yaml

@@ -148,6 +148,13 @@ jobs:
         if: matrix.chart == 'service'
         run: bash deployments/charts/service/ci/validate-mcp-chart.sh
 
+      - name: Chart-specific render tests
+        run: |
+          TEST_SCRIPT="deployments/charts/${{ matrix.chart }}/tests/render-tests.sh"
+          if [ -f "$TEST_SCRIPT" ]; then
+            bash "$TEST_SCRIPT"
+          fi
+
       - name: Validate unified OSMO chart
         if: matrix.chart == 'osmo'
         run: bash deployments/charts/osmo/tests/test_osmo_charts.sh osmo

.github/workflows/pr-checks.yaml

@@ -163,7 +163,7 @@ jobs:
         ports:
           - 2375:2375
     steps:
-      - name: Install Node.js, Docker CLI, and dependencies
+      - name: Install Node.js, Docker CLI, kubectl, and dependencies
         run: |
           apt-get update
           apt-get install -y ca-certificates curl gnupg git-lfs
@@ -183,6 +183,15 @@ jobs:
           # Install Docker CLI (client only, connects to DinD service)
           apt-get install -y docker-ce-cli
 
+          # Install kubectl for deployment-script tests that render Secrets.
+          KUBECTL_VERSION=v1.31.0
+          curl -fsSLo /tmp/kubectl \
+            "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
+          curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" \
+            | awk '{print $1"  /tmp/kubectl"}' | sha256sum -c -
+          install -m 0755 /tmp/kubectl /usr/local/bin/kubectl
+          rm -f /tmp/kubectl
+
           # Remove Debian's nodejs package if present and install from NodeSource
           apt-get remove -y nodejs || true
           apt-get install -y nodejs=20.*
@@ -191,6 +200,7 @@ jobs:
           echo "Node.js version: $(node --version)"
           echo "npm version: $(npm --version)"
           echo "Docker version: $(docker --version)"
+          kubectl version --client --output=yaml
           echo "Git LFS version: $(git-lfs --version)"
 
       - name: Clean up stale LFS hooks

AGENTS.md

@@ -85,7 +85,7 @@ Entry point: `service/core/service.py`. Framework: FastAPI + Uvicorn + OpenTelem
 
 | Submodule | Purpose |
 |-----------|---------|
-| `auth/` | JWT token lifecycle, access token CRUD, user management, role assignment |
+| `auth/` | JWT token lifecycle, access token CRUD, user management, role assignment, and Kubernetes Secret-backed backend bootstrap authentication |
 | `workflow/` | Workflow submit/list/cancel, resource quota, pool allocation, task coordination, credential management |
 | `config/` | Service/workflow configuration CRUD with versioning and history. Pod templates, resource validation rules, pool/backend config. |
 | `data/` | Workflow data and storage operations built on multi-backend storage. |
@@ -94,6 +94,17 @@ Entry point: `service/core/service.py`. Framework: FastAPI + Uvicorn + OpenTelem
 
 **Error types**: Defined in `lib/utils/` — see the `OSMOError` hierarchy for the full list.
 
+Backend bootstrap authentication is implemented by
+`auth/backend_secret_auth.py`, which maps mounted Kubernetes Secret tokens to
+the fixed `osmo-backend` identity. The service Helm chart creates managed
+development credentials with a short-lived kubectl hook without rendering token
+material in Helm output. Changes to this authentication path must run:
+
+```bash
+bazel test //src/service/core/auth/tests:test_backend_secret_auth
+bash deployments/charts/service/tests/render-tests.sh
+```
+
 ### Supporting Services
 
 | Service | Purpose |

deployments/BUILD

@@ -15,3 +15,8 @@ limitations under the License.
 
 SPDX-License-Identifier: Apache-2.0
 """
+
+exports_files([
+    "scripts/common.sh",
+    "scripts/deploy-k8s.sh",
+])

deployments/README.md

@@ -100,6 +100,8 @@ For users who already have Kubernetes infrastructure and want to deploy OSMO dir
 > - Kubernetes namespaces (`osmo-minimal`, `osmo-operator`, `osmo-workflows`)
 > - Database secrets (`db-secret` with PostgreSQL password)
 > - Redis secrets (`redis-secret` with Redis password)
+> - For production or multi-cluster installs, a backend bootstrap token Secret
+>   copied to the control and compute clusters
 > - MEK ConfigMap (Master Encryption Key)
 > - The PostgreSQL database itself
 >
@@ -124,10 +126,10 @@ For a direct Helm install, deploy the charts in this order:
 ```bash
 kubectl create namespace osmo --dry-run=client -o yaml | kubectl apply -f -
 kubectl create namespace osmo-test --dry-run=client -o yaml | kubectl apply -f -
-BACKEND_OPERATOR_PASSWORD=$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 | tr -d '\n=' | head -c 43)
-kubectl create secret generic backend-operator-password \
+LOCAL_ADMIN_PASSWORD=$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 | tr -d '\n=' | head -c 43)
+kubectl create secret generic local-admin-password \
   --namespace osmo \
-  --from-literal=password="$BACKEND_OPERATOR_PASSWORD" \
+  --from-literal=password="$LOCAL_ADMIN_PASSWORD" \
   --dry-run=client -o yaml | kubectl apply -f -
 
 if ! kubectl get configmap mek-config --namespace osmo >/dev/null 2>&1; then
@@ -156,6 +158,11 @@ helm upgrade --install osmo-backend-operator osmo/backend-operator \
   --wait
 ```
 
+For the local quick-start only, the service values generate
+`backend-operator-token` during the first Helm install. Because the backend
+operator is installed in the same namespace, it consumes that Secret directly;
+no pre-created backend Secret is required.
+
 After installing the CLI and logging in, set the demo pool and LocalStack data credential:
 
 ```bash

deployments/charts/README.md

@@ -32,16 +32,17 @@ The former quick-start values are preserved as chart-specific values files:
 - `service/quick-start-values.yaml`
 - `backend-operator/quick-start-values.yaml`
 
-Create the namespaces, the local backend-operator password secret, and the MEK
-ConfigMap used by the service pods:
+Create the namespaces, the local admin password Secret, and the MEK ConfigMap
+used by the service pods. The service quick-start values create the shared
+backend bootstrap Secret through the chart-managed development mode:
 
 ```bash
 kubectl create namespace osmo --dry-run=client -o yaml | kubectl apply -f -
 kubectl create namespace osmo-test --dry-run=client -o yaml | kubectl apply -f -
-BACKEND_OPERATOR_PASSWORD=$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 | tr -d '\n=' | head -c 43)
-kubectl create secret generic backend-operator-password \
+LOCAL_ADMIN_PASSWORD=$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 | tr -d '\n=' | head -c 43)
+kubectl create secret generic local-admin-password \
   --namespace osmo \
-  --from-literal=password="$BACKEND_OPERATOR_PASSWORD" \
+  --from-literal=password="$LOCAL_ADMIN_PASSWORD" \
   --dry-run=client -o yaml | kubectl apply -f -
 
 if ! kubectl get configmap mek-config --namespace osmo >/dev/null 2>&1; then
@@ -121,4 +122,9 @@ For production, use your environment-specific values instead of the local quick-
 - Provide managed PostgreSQL, Redis, and object storage settings or enable the chart-managed development dependencies only for non-production use.
 - Enable OAuth2/authz in the service chart when exposing OSMO to untrusted networks.
 - Configure `backend-operator.global.serviceUrl` to the service gateway URL reachable from the backend cluster.
-- Use `backend-operator.global.loginMethod` with either password or token credentials stored in Kubernetes Secrets.
+- Provision one backend bootstrap Secret per compute plane in both the control
+  and compute clusters. Configure the service chart's
+  `services.backendApiTokens.credentials[].existingSecret.name`
+  and `backend-operator.global.accountTokenSecret` to consume the matching Secret.
+  The service chart's `managedSecret` mode is intended only for single-cluster
+  development where the backend operator can consume the namespace-local Secret.

deployments/charts/backend-operator/README.md

@@ -57,6 +57,12 @@ helm upgrade --install osmo-backend-operator osmo/backend-operator \
 | `global.enableClusterRoles` | Enable cluster roles | `true` |
 | `global.enableNonClusterRoles` | Enable non-cluster roles | `true` |
 
+For Secret-backed bootstrap, set `global.loginMethod: token` and reference the
+copy of the credential that is also mounted by the control-plane service chart.
+The token volume is mounted as a directory so Kubernetes Secret projection
+updates are visible. During rotation, restart both backend deployments before
+the control plane stops accepting the previous token.
+
 ### Global NetworkPolicy Settings
 
 When enabled, a `NetworkPolicy` is applied to the workflow namespace (`global.backendNamespace`) that allows unrestricted external internet egress while blocking cross-namespace cluster traffic except to explicitly allowlisted namespaces.

deployments/charts/backend-operator/quick-start-values.yaml

@@ -15,7 +15,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Local-development values preserved from the former quick-start install path.
-# Install after the service chart has created the backend-operator admin user.
+# Install after the service chart creates backend-operator-token through its
+# chart-managed backend credential mode.
 
 global:
   osmoImageLocation: nvcr.io/nvidia/osmo
@@ -26,10 +27,9 @@ global:
   backendNamespace: default
   backendTestNamespace: osmo-test
   backendName: default
-  accountUsername: backend-operator
-  accountPasswordSecret: backend-operator-password
-  accountPasswordSecretKey: password
-  loginMethod: password
+  accountTokenSecret: backend-operator-token
+  accountTokenSecretKey: token
+  loginMethod: token
   nodeSelector:
     node_group: service
 

deployments/charts/backend-operator/templates/backend-listener.yaml

@@ -138,8 +138,7 @@ spec:
           readOnly: true
         {{- else if eq .Values.global.loginMethod "token" }}
         - name: osmo-secret
-          mountPath: /opt/osmo/secrets/token.txt
-          subPath: token.txt
+          mountPath: /opt/osmo/secrets
           readOnly: true
         {{- end }}
         {{- if .Values.services.backendListener.volumeMounts }}

deployments/charts/backend-operator/templates/backend-worker.yaml

@@ -136,8 +136,7 @@ spec:
           readOnly: true
         {{- else if eq .Values.global.loginMethod "token" }}
         - name: osmo-secret
-          mountPath: /opt/osmo/secrets/token.txt
-          subPath: token.txt
+          mountPath: /opt/osmo/secrets
           readOnly: true
         {{- end }}
         {{- if .Values.backendTestRunner.enabled }}