deploy: the cluster issuer is a workload issuer; --node-retention per environment

The testnets listed the GKE service account issuer under --issuer, next to
the human one. A service account token therefore enrolled as a human: a
90-day session, and the token accepted at /user/*. The probe and canary
cronjobs enroll a fresh identity on every run, so bananas held 19,601
admitted rows with sessions of ~2160h, none of them a member any more.
Under --workload-issuer, which the Kubernetes guide already prescribes,
these are 48h sessions renewed with a fresh projected token, and the
control plane's node GC collects the rows of the runs that ended.

--node-retention is rendered from the NODE_RETENTION environment variable
(unset is the binary's 720h), so an environment that runs scale tests can
set it to days and watch the sweep within the test window.

Author: aojea

.github/k8s/agentmesh-control-plane-template.yaml

@@ -128,10 +128,14 @@ spec:
         args:
         - "--bind-address=0.0.0.0:8080"
         - "--db-driver=postgres"
-        - "--issuer=https://auth.sam-mesh.dev,https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"
+        - "--issuer=https://auth.sam-mesh.dev"
+        # The cluster's service account tokens are workload identities: a
+        # 48h session renewed with a fresh token, no /user/* access.
+        - "--workload-issuer=https://container.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${CLUSTER_REGION}/clusters/${CLUSTER_NAME}"
         - "--allowed-audiences=agentmesh-audience,agentmesh-control-plane-audience,sam-mesh-audience,sam-control-plane-audience"
         - "--auto-approve-enrollment"
         - "--key-grace-period=48h"
+        - "--node-retention=${NODE_RETENTION}"
         resources:
           requests:
             cpu: 100m

.github/workflows/deploy.yaml

@@ -361,6 +361,10 @@ jobs:
           # routers per member, six routers hold 10,000 members with one
           # router out for a rollout.
           VAR_ROUTER_HIGH_WATERMARK: ${{ vars.ROUTER_HIGH_WATERMARK }}
+          # How long the control plane keeps a lapsed or silent member's
+          # record. Unset is the binary's default, 720h; an environment that
+          # runs scale tests sets it to days so the sweep is observable.
+          VAR_NODE_RETENTION: ${{ vars.NODE_RETENTION }}
         run: |
           print_rollout_diagnostics() {
             local namespace="$1"
@@ -395,8 +399,9 @@ jobs:
           export CLUSTER_NAME="${VAR_CLUSTER_NAME}"
           export ROUTER_REPLICAS="${VAR_ROUTER_REPLICAS:-3}"
           export ROUTER_HIGH_WATERMARK="${VAR_ROUTER_HIGH_WATERMARK:-6000}"
+          export NODE_RETENTION="${VAR_NODE_RETENTION:-720h}"
 
-          envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/agentmesh-control-plane-template.yaml | kubectl apply -f -
+          envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG} ${NODE_RETENTION}' < .github/k8s/agentmesh-control-plane-template.yaml | kubectl apply -f -
           envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG} ${ROUTER_REPLICAS} ${ROUTER_HIGH_WATERMARK}' < .github/k8s/agentmesh-router-template.yaml | kubectl apply -f -
           envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID} ${CLUSTER_NAME} ${CLUSTER_REGION} ${IMAGE_TAG}' < .github/k8s/agentmesh-console-template.yaml | kubectl apply -f -
           envsubst '${ENV_NAME} ${NAMESPACE} ${GCP_PROJECT_ID}' < .github/k8s/dns-sync-cronjob-template.yaml | kubectl apply -f -