From 9d51e960920b77730cc9f834169be5b628e2011a Mon Sep 17 00:00:00 2001
From: Jason Roy
Date: Thu, 29 Oct 2026 20:43:18 -0700
Subject: [PATCH 1/4] test: add three-pod kind invalidation burst gate
---
.github/workflows/multi-pod-e2e.yml | 77 +++++++++
.../app/api/e2e/multi-pod/route.ts | 22 +++
.../app/e2e/multi-pod/[id]/page.tsx | 36 +++++
apps/e2e-test-app/next.config.mjs | 6 +-
docs/development.md | 49 ++++++
k8s/e2e/Dockerfile | 28 ++++
k8s/e2e/Dockerfile.dockerignore | 6 +
k8s/e2e/cluster.yaml | 89 ++++++++++
k8s/e2e/diagnostics.sh | 12 ++
k8s/e2e/driver.mjs | 153 ++++++++++++++++++
k8s/e2e/job.yaml | 25 +++
k8s/e2e/run.sh | 28 ++++
12 files changed, 530 insertions(+), 1 deletion(-)
create mode 100644 .github/workflows/multi-pod-e2e.yml
create mode 100644 apps/e2e-test-app/app/api/e2e/multi-pod/route.ts
create mode 100644 apps/e2e-test-app/app/e2e/multi-pod/[id]/page.tsx
create mode 100644 k8s/e2e/Dockerfile
create mode 100644 k8s/e2e/Dockerfile.dockerignore
create mode 100644 k8s/e2e/cluster.yaml
create mode 100644 k8s/e2e/diagnostics.sh
create mode 100644 k8s/e2e/driver.mjs
create mode 100644 k8s/e2e/job.yaml
create mode 100644 k8s/e2e/run.sh
diff --git a/.github/workflows/multi-pod-e2e.yml b/.github/workflows/multi-pod-e2e.yml
new file mode 100644
index 0000000..e712c0f
--- /dev/null
+++ b/.github/workflows/multi-pod-e2e.yml
@@ -0,0 +1,77 @@
+name: Multi-pod E2E
+
+on:
+ pull_request:
+ push:
+ branches: [main]
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ multi-pod:
+ name: Multi-pod (${{ matrix.storage }}, ${{ matrix.control }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ strategy:
+ fail-fast: false
+ matrix:
+ storage: [memory, redis]
+ control: [fixed, reverted]
+ steps:
+ - uses: actions/checkout@v4
+ # Temporary until #106 lands: build the fixture with the pending rc.6 handler.
+ - uses: actions/checkout@v4
+ with:
+ ref: mrjasonroy/empowering-pumps-ep-59
+ path: .handler-under-test
+ - uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+ - name: Build production standalone fixture
+ env:
+ STORAGE: ${{ matrix.storage }}
+ CONTROL: ${{ matrix.control }}
+ run: |
+ revert=0
+ if [ "$CONTROL" = reverted ]; then revert=1; fi
+ docker build -f k8s/e2e/Dockerfile --build-context handler=.handler-under-test \
+ --build-arg USE_CACHE="$STORAGE" --build-arg REVERT_TIMESTAMP="$revert" \
+ -t cache-e2e:pr .
+ - name: Create kind cluster
+ uses: helm/kind-action@v1.12.0
+ with:
+ cluster_name: cache-e2e
+ - name: Load fixture
+ run: kind load docker-image cache-e2e:pr --name cache-e2e
+ - name: Single save and five CMS bursts
+ env:
+ CONTROL: ${{ matrix.control }}
+ run: |
+ mkdir -p artifacts
+ set +e
+ bash k8s/e2e/run.sh 2>&1 | tee artifacts/driver.txt
+ result=${PIPESTATUS[0]}
+ set -e
+ if [ "$CONTROL" = reverted ]; then
+ # Infrastructure/build failures are not evidence that the regression was caught.
+ test "$result" -ne 0
+ grep -q 'STALE_CONTENT' artifacts/driver.txt
+ echo 'Negative control caught stale content as required.'
+ else
+ test "$result" -eq 0
+ grep -q 'MULTI_POD_PASS' artifacts/driver.txt
+ fi
+ - name: Collect pod logs and Redis tag records
+ if: always()
+ run: bash k8s/e2e/diagnostics.sh
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: multi-pod-${{ matrix.storage }}-${{ matrix.control }}
+ path: artifacts/
+ retention-days: 7
+ - name: Remove disposable cluster
+ if: always()
+ run: kind delete cluster --name cache-e2e
diff --git a/apps/e2e-test-app/app/api/e2e/multi-pod/route.ts b/apps/e2e-test-app/app/api/e2e/multi-pod/route.ts
new file mode 100644
index 0000000..3eadbb0
--- /dev/null
+++ b/apps/e2e-test-app/app/api/e2e/multi-pod/route.ts
@@ -0,0 +1,22 @@
+import { revalidateTag } from "next/cache";
+import { type NextRequest, NextResponse } from "next/server";
+
+/** Test-only CMS webhook; the kind Secret supplies its bearer token. */
+export async function POST(request: NextRequest) {
+ const secret = process.env.E2E_REVALIDATE_SECRET;
+ if (!secret || request.headers.get("authorization") !== `Bearer ${secret}`) {
+ return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+ }
+ const body: unknown = await request.json().catch(() => null);
+ const tags = body && typeof body === "object" && "tags" in body ? body.tags : null;
+ if (
+ !Array.isArray(tags) ||
+ tags.length === 0 ||
+ tags.length > 19 ||
+ !tags.every((tag): tag is string => typeof tag === "string" && /^multi-pod-[\w-]+$/.test(tag))
+ ) {
+ return NextResponse.json({ error: "Expected 1–19 multi-pod tags" }, { status: 400 });
+ }
+ for (const tag of tags) revalidateTag(tag, { expire: 0 });
+ return NextResponse.json({ pod: process.env.POD_NAME, tags });
+}
diff --git a/apps/e2e-test-app/app/e2e/multi-pod/[id]/page.tsx b/apps/e2e-test-app/app/e2e/multi-pod/[id]/page.tsx
new file mode 100644
index 0000000..0ca2f91
--- /dev/null
+++ b/apps/e2e-test-app/app/e2e/multi-pod/[id]/page.tsx
@@ -0,0 +1,36 @@
+import { cacheLife, cacheTag } from "next/cache";
+import { cookies } from "next/headers";
+import { Suspense } from "react";
+
+export function generateStaticParams() {
+ return [{ id: "build" }];
+}
+
+async function Snapshot({ id }: { id: string }) {
+ "use cache";
+ cacheLife({ stale: 60, revalidate: 86400, expire: 172800 });
+ cacheTag(`multi-pod-${id}`);
+ if (id === "build") return build
;
+ const origin = process.env.E2E_ORIGIN_URL;
+ if (!origin) throw new Error("E2E_ORIGIN_URL is required for the multi-pod fixture");
+ const response = await fetch(`${origin}/${encodeURIComponent(id)}`);
+ if (!response.ok) throw new Error("Multi-pod fixture origin failed");
+ return {await response.text()}
;
+}
+
+async function RequestPart() {
+ await cookies();
+ return Request content
;
+}
+
+export default async function Page({ params }: { params: Promise<{ id: string }> }) {
+ const { id } = await params;
+ return (
+
+
+ Loading
}>
+
+
+
+ );
+}
diff --git a/apps/e2e-test-app/next.config.mjs b/apps/e2e-test-app/next.config.mjs
index b968812..2f7c7b2 100644
--- a/apps/e2e-test-app/next.config.mjs
+++ b/apps/e2e-test-app/next.config.mjs
@@ -19,6 +19,7 @@ const handlers =
/** @type {import('next').NextConfig} */
const nextConfig = {
cacheComponents: true,
+ ...(process.env.E2E_STANDALONE === "1" ? { output: "standalone" } : {}),
// Custom cacheLife profiles for testing
cacheLife: {
@@ -51,4 +52,7 @@ const config16_4_2 = {
export default process.env.E2E_CONFIG === "16.4.2"
? config16_4_2
- : withCache(nextConfig, { handlers });
+ : withCache(nextConfig, {
+ handlers,
+ ...(process.env.E2E_USE_CACHE ? { useCache: process.env.E2E_USE_CACHE } : {}),
+ });
diff --git a/docs/development.md b/docs/development.md
index ba6468b..8c9bde3 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -195,3 +195,52 @@ npm. See [auto-release.md](./auto-release.md) and
- **Next.js deploy tests**: Next.js's own caching suites against the package, on PRs that touch the package source
- **Publish**: Runs on version tags (tests + npm publish)
- **Version Check**: Runs daily to check for new Next.js versions
+
+### Three-pod invalidation gate
+
+`multi-pod-e2e.yml` builds the PR's production standalone fixture and loads it into a disposable
+kind cluster: three app replicas behind a ClusterIP Service, one Redis, and a driver Job that
+also acts as the fixture CMS origin. Both default memory storage and `useCache: "redis"` run.
+The driver discovers all three ready Pod IPs from the Service Endpoints; it does not use a
+Service port-forward, which would pin requests to one pod.
+
+The driver warms 19 tagged pages on every pod, changes the CMS versions, sends one authenticated
+webhook, and checks the first subsequent read of every page on every pod. It then repeats five
+bursts: force regeneration, hold all 19 pre-save origin snapshots, change the CMS versions,
+send 19 invalidations spaced 150 ms apart through the Service, observe their commits in Redis,
+release the renders, and check every page directly on all three pods. Freshness assertions never
+retry. Only setup and invalidation-commit barriers poll. The bearer token is generated for each
+cluster and injected through a Kubernetes Secret.
+
+Separate negative-control jobs build a throwaway image with the #104 page timestamp assignment
+reverted. They must fail specifically with `STALE_CONTENT`; build, networking or timeout errors
+do not satisfy the negative control. `REVERT_TIMESTAMP` defaults to `0` and never edits source
+in the checkout. All four jobs have a ten-minute limit. Artifacts include the driver transcript,
+pod logs/events, and Redis tag hash/index dumps, including on expected negative-control failures.
+
+To reproduce with Docker, kind, kubectl, Node 22 and OpenSSL installed:
+
+```bash
+docker build -f k8s/e2e/Dockerfile --build-context handler=. \
+ --build-arg USE_CACHE=redis -t cache-e2e:pr .
+kind create cluster --name cache-e2e
+kind load docker-image cache-e2e:pr --name cache-e2e
+bash k8s/e2e/run.sh
+bash k8s/e2e/diagnostics.sh
+kind delete cluster --name cache-e2e
+```
+
+For the negative control add `--build-arg REVERT_TIMESTAMP=1`; the driver must exit nonzero
+with `STALE_CONTENT`. Use a fresh cluster for each run.
+
+After the workflow has run, the repository owner should add these exact status checks to the
+`main` branch protection/ruleset alongside the existing required checks:
+
+- `Multi-pod (memory, fixed)`
+- `Multi-pod (redis, fixed)`
+- `Multi-pod (memory, reverted)`
+- `Multi-pod (redis, reverted)`
+
+Select GitHub Actions as their source. Do not make the negative-control failure itself a required
+failed check: its job succeeds only when it observes the specific expected regression. Workflow
+changes do not change branch protection automatically.
diff --git a/k8s/e2e/Dockerfile b/k8s/e2e/Dockerfile
new file mode 100644
index 0000000..c808666
--- /dev/null
+++ b/k8s/e2e/Dockerfile
@@ -0,0 +1,28 @@
+FROM node:22-bookworm-slim AS build
+WORKDIR /app
+RUN corepack enable
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY packages/cache-handler/package.json packages/cache-handler/package.json
+COPY apps/e2e-test-app/package.json apps/e2e-test-app/package.json
+RUN pnpm install --frozen-lockfile
+COPY --from=handler packages/cache-handler packages/cache-handler
+COPY apps/e2e-test-app apps/e2e-test-app
+# Negative control lives only in this throwaway image; never changes the checkout.
+ARG REVERT_TIMESTAMP=0
+RUN if [ "$REVERT_TIMESTAMP" = 1 ]; then \
+ sed -i 's/lastModified: this.renderStarts.get(cacheKey) ?? nowMs()/lastModified: nowMs()/' packages/cache-handler/src/handlers/redis.ts; \
+ fi
+RUN pnpm --filter @mrjasonroy/cache-components-cache-handler build
+ARG USE_CACHE=memory
+ENV E2E_USE_CACHE=$USE_CACHE
+ENV E2E_STANDALONE=1 NEXT_TELEMETRY_DISABLED=1 REDIS_URL=redis://redis:6379
+RUN pnpm --filter e2e-test-app build
+
+FROM node:22-bookworm-slim
+WORKDIR /app
+ENV NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 NEXT_TELEMETRY_DISABLED=1
+COPY --from=build /app/apps/e2e-test-app/.next/standalone ./
+COPY --from=build /app/apps/e2e-test-app/.next/static ./apps/e2e-test-app/.next/static
+COPY k8s/e2e/driver.mjs ./apps/e2e-test-app/driver.mjs
+USER node
+CMD ["node", "apps/e2e-test-app/server.js"]
diff --git a/k8s/e2e/Dockerfile.dockerignore b/k8s/e2e/Dockerfile.dockerignore
new file mode 100644
index 0000000..dcba173
--- /dev/null
+++ b/k8s/e2e/Dockerfile.dockerignore
@@ -0,0 +1,6 @@
+**/node_modules
+**/.next
+**/dist
+**/.turbo
+.git
+drafts
diff --git a/k8s/e2e/cluster.yaml b/k8s/e2e/cluster.yaml
new file mode 100644
index 0000000..f3b45ad
--- /dev/null
+++ b/k8s/e2e/cluster.yaml
@@ -0,0 +1,89 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: cache-e2e
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: redis
+ namespace: cache-e2e
+spec:
+ replicas: 1
+ selector:
+ matchLabels: { app: redis }
+ template:
+ metadata:
+ labels: { app: redis }
+ spec:
+ containers:
+ - name: redis
+ image: redis:7.4-alpine
+ args: ["--maxmemory-policy", "noeviction"]
+ ports: [{ containerPort: 6379 }]
+ readinessProbe:
+ exec:
+ command: ["redis-cli", "ping"]
+ periodSeconds: 1
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: redis
+ namespace: cache-e2e
+spec:
+ selector: { app: redis }
+ ports: [{ port: 6379, targetPort: 6379 }]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: app
+ namespace: cache-e2e
+spec:
+ replicas: 3
+ selector:
+ matchLabels: { app: cache-app }
+ template:
+ metadata:
+ labels: { app: cache-app }
+ spec:
+ containers:
+ - name: app
+ image: cache-e2e:pr
+ imagePullPolicy: Never
+ env:
+ - name: POD_NAME
+ valueFrom:
+ fieldRef: { fieldPath: metadata.name }
+ - { name: REDIS_URL, value: "redis://redis:6379" }
+ - { name: CACHE_KEY_PREFIX, value: "multi-pod:" }
+ - { name: E2E_ORIGIN_URL, value: "http://driver:8080" }
+ - name: E2E_REVALIDATE_SECRET
+ valueFrom:
+ secretKeyRef: { name: revalidate, key: token }
+ ports: [{ containerPort: 3000 }]
+ readinessProbe:
+ tcpSocket: { port: 3000 }
+ periodSeconds: 1
+ resources:
+ requests: { cpu: "100m", memory: "128Mi" }
+ limits: { memory: "768Mi" }
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: app
+ namespace: cache-e2e
+spec:
+ selector: { app: cache-app }
+ ports: [{ port: 3000, targetPort: 3000 }]
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: driver
+ namespace: cache-e2e
+spec:
+ selector: { job-name: driver }
+ ports: [{ port: 8080, targetPort: 8080 }]
diff --git a/k8s/e2e/diagnostics.sh b/k8s/e2e/diagnostics.sh
new file mode 100644
index 0000000..a9f8011
--- /dev/null
+++ b/k8s/e2e/diagnostics.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -u
+mkdir -p artifacts
+kubectl -n cache-e2e get pods -o wide > artifacts/pods.txt 2>&1
+kubectl -n cache-e2e describe pods > artifacts/describe.txt 2>&1
+kubectl -n cache-e2e get events --sort-by=.metadata.creationTimestamp > artifacts/events.txt 2>&1
+for pod in $(kubectl -n cache-e2e get pods -o name); do
+ kubectl -n cache-e2e logs "$pod" --all-containers > "artifacts/${pod#pod/}.log" 2>&1
+ kubectl -n cache-e2e logs "$pod" --all-containers --previous > "artifacts/${pod#pod/}-previous.log" 2>&1 || true
+done
+kubectl -n cache-e2e exec deployment/redis -- redis-cli HGETALL multi-pod:tags > artifacts/redis-tags.txt 2>&1
+kubectl -n cache-e2e exec deployment/redis -- redis-cli ZRANGE multi-pod:tag-index 0 -1 WITHSCORES > artifacts/redis-tag-index.txt 2>&1
diff --git a/k8s/e2e/driver.mjs b/k8s/e2e/driver.mjs
new file mode 100644
index 0000000..a870fd8
--- /dev/null
+++ b/k8s/e2e/driver.mjs
@@ -0,0 +1,153 @@
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { setTimeout as sleep } from "node:timers/promises";
+import Redis from "ioredis";
+
+const pods = process.env.POD_URLS?.split(",") ?? [];
+assert.equal(new Set(pods).size, 3, "Exactly three distinct ready pod addresses required");
+const service = process.env.APP_URL;
+const secret = process.env.E2E_REVALIDATE_SECRET;
+assert.ok(service && secret, "APP_URL and E2E_REVALIDATE_SECRET required");
+const redis = new Redis(process.env.REDIS_URL, {
+ maxRetriesPerRequest: 0,
+ retryStrategy: () => null,
+});
+redis.on("error", (error) => process.stderr.write(`${error.message}\n`));
+const ids = Array.from({ length: 19 }, (_, i) => `article-${i}`);
+const tags = ids.map((id) => `multi-pod-${id}`);
+const state = new Map(ids.map((id) => [id, { version: 1, hold: null, started: false }]));
+const server = createServer(async (request, response) => {
+ const entry = state.get(request.url?.slice(1));
+ if (!entry) {
+ response.writeHead(404).end();
+ return;
+ }
+ const snapshot = entry.version;
+ entry.started = true;
+ await entry.hold;
+ response.end(String(snapshot));
+});
+const report = (message) => process.stdout.write(`${message}\n`);
+
+async function until(check, label) {
+ const deadline = Date.now() + 15000;
+ while (!(await check())) {
+ assert.ok(Date.now() < deadline, `Timed out: ${label}`);
+ await sleep(20);
+ }
+}
+
+async function read(base, id) {
+ const response = await fetch(`${base}/e2e/multi-pod/${id}`, {
+ signal: AbortSignal.timeout(30000),
+ });
+ assert.equal(response.status, 200, `${base}/${id}`);
+ const html = await response.text();
+ const match = html.match(/data-testid="version"[^>]*>(\d+));
+ assert.ok(match, `Missing version in ${base}/${id}: ${html.slice(0, 200)}`);
+ return Number(match[1]);
+}
+
+async function invalidate(selected) {
+ const before = await redis.hmget("multi-pod:tags", ...selected);
+ const response = await fetch(`${service}/api/e2e/multi-pod`, {
+ method: "POST",
+ // A new connection makes the ClusterIP select a backend per webhook.
+ headers: {
+ authorization: `Bearer ${secret}`,
+ "content-type": "application/json",
+ connection: "close",
+ },
+ body: JSON.stringify({ tags: selected }),
+ signal: AbortSignal.timeout(10000),
+ });
+ assert.equal(response.status, 200, "CMS webhook status");
+ const { pod } = await response.json();
+ // HTTP headers can precede Next's pending tag writes. Observe Redis before releasing renders.
+ await until(async () => {
+ const after = await redis.hmget("multi-pod:tags", ...selected);
+ return after.every((record, i) => record && record !== before[i]);
+ }, "invalidation committed in Redis");
+ return pod;
+}
+
+async function freshEverywhere(version, phase) {
+ for (const pod of pods) {
+ for (const id of ids) {
+ const actual = await read(pod, id);
+ // Never retry freshness: the first read after invalidation is the contract.
+ assert.equal(
+ actual,
+ version,
+ `STALE_CONTENT ${phase} ${pod}/${id}: expected ${version}, got ${actual}`,
+ );
+ }
+ }
+ report(`${phase}: ${ids.length} pages × ${pods.length} pods fresh at version ${version}`);
+}
+
+let release = () => {};
+try {
+ await new Promise((resolve) => server.listen(8080, "0.0.0.0", resolve));
+ const unauthorized = await fetch(`${service}/api/e2e/multi-pod`, {
+ method: "POST",
+ body: JSON.stringify({ tags }),
+ });
+ assert.equal(unauthorized.status, 401, "Webhook must require the Secret");
+ await freshEverywhere(1, "warm");
+ // One CMS save and one webhook, followed by the first read on every replica.
+ for (const entry of state.values()) entry.version = 2;
+ await invalidate(tags);
+ await freshEverywhere(2, "single save");
+
+ for (let round = 1; round <= 5; round++) {
+ const version = round + 2;
+ const held = new Promise((resolve) => {
+ release = resolve;
+ });
+ for (const entry of state.values()) {
+ entry.hold = held;
+ entry.started = false;
+ }
+ // Evict warm shells before starting controlled renders of the pre-save snapshot.
+ await invalidate(tags);
+ const renders = ids.map((id, i) => read(pods[(i + round) % pods.length], id));
+ // Attach rejection handlers immediately while the renders are deliberately held.
+ const finished = Promise.all(renders);
+ finished.catch(() => {});
+ await until(
+ () => [...state.values()].every((entry) => entry.started),
+ "all origin snapshots held",
+ );
+ for (const entry of state.values()) entry.version = version;
+ const receivers = new Set();
+ const start = Date.now();
+ const burst = tags.map(async (tag, i) => {
+ await sleep(i * 150);
+ receivers.add(await invalidate([tag]));
+ });
+ await Promise.all(burst);
+ report(
+ `round ${round}: 19 invalidations committed in ${Date.now() - start}ms; receivers=${[...receivers]}`,
+ );
+ release();
+ const snapshots = await finished;
+ assert.ok(
+ snapshots.every((snapshot) => snapshot === version - 1),
+ "Held renders must contain pre-save data",
+ );
+ for (const entry of state.values()) entry.hold = null;
+ // Let the completed shell writes settle; never poll the page to hide a stale first read.
+ await sleep(200);
+ await freshEverywhere(version, `burst ${round}`);
+ }
+ report("MULTI_POD_PASS: single save and five held-render bursts");
+} catch (error) {
+ process.stderr.write(`${error.stack ?? error}\n`);
+ process.exitCode = 1;
+} finally {
+ release();
+ server.closeAllConnections();
+ await new Promise((resolve) => server.close(resolve));
+ redis.disconnect();
+}
diff --git a/k8s/e2e/job.yaml b/k8s/e2e/job.yaml
new file mode 100644
index 0000000..6dfed8e
--- /dev/null
+++ b/k8s/e2e/job.yaml
@@ -0,0 +1,25 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: driver
+ namespace: cache-e2e
+spec:
+ backoffLimit: 0
+ activeDeadlineSeconds: 180
+ template:
+ spec:
+ restartPolicy: Never
+ containers:
+ - name: driver
+ image: cache-e2e:pr
+ imagePullPolicy: Never
+ command: ["node", "apps/e2e-test-app/driver.mjs"]
+ env:
+ - { name: REDIS_URL, value: "redis://redis:6379" }
+ - { name: APP_URL, value: "http://app:3000" }
+ - name: POD_URLS
+ valueFrom:
+ configMapKeyRef: { name: pods, key: urls }
+ - name: E2E_REVALIDATE_SECRET
+ valueFrom:
+ secretKeyRef: { name: revalidate, key: token }
diff --git a/k8s/e2e/run.sh b/k8s/e2e/run.sh
new file mode 100644
index 0000000..7bd58bf
--- /dev/null
+++ b/k8s/e2e/run.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+kubectl apply -f k8s/e2e/cluster.yaml
+kubectl -n cache-e2e create secret generic revalidate --from-literal=token="$(openssl rand -hex 24)"
+kubectl -n cache-e2e rollout status deployment/redis --timeout=90s
+kubectl -n cache-e2e rollout status deployment/app --timeout=90s
+# Individual Pod IPs, not port-forwarding a Service (which pins one backend).
+pods=$(kubectl -n cache-e2e get endpoints app -o json | node -e '
+let input = "";
+process.stdin.on("data", chunk => input += chunk);
+process.stdin.on("end", () => {
+ const addresses = JSON.parse(input).subsets.flatMap(s => s.addresses ?? []);
+ if (addresses.length !== 3) process.exit(1);
+ process.stdout.write(addresses.map(a => `http://${a.ip}:3000`).join(","));
+});')
+kubectl -n cache-e2e create configmap pods --from-literal=urls="$pods"
+kubectl apply -f k8s/e2e/job.yaml
+for _ in $(seq 1 180); do
+ status=$(kubectl -n cache-e2e get job driver -o jsonpath='{.status.succeeded},{.status.failed}')
+ case "$status" in
+ 1,*) kubectl -n cache-e2e logs job/driver; exit 0 ;;
+ *,1) kubectl -n cache-e2e logs job/driver; exit 1 ;;
+ esac
+ sleep 1
+done
+kubectl -n cache-e2e logs job/driver || true
+echo 'Driver did not finish in 180 seconds' >&2
+exit 1
From b15b0a42b16bde2c9a6cbb413323b21944c584d0 Mon Sep 17 00:00:00 2001
From: Jason Roy
Date: Thu, 29 Oct 2026 20:46:47 -0700
Subject: [PATCH 2/4] test: include lazy Redis dependency in standalone fixture
---
k8s/e2e/Dockerfile | 3 +++
1 file changed, 3 insertions(+)
diff --git a/k8s/e2e/Dockerfile b/k8s/e2e/Dockerfile
index c808666..75f794e 100644
--- a/k8s/e2e/Dockerfile
+++ b/k8s/e2e/Dockerfile
@@ -17,11 +17,14 @@ ARG USE_CACHE=memory
ENV E2E_USE_CACHE=$USE_CACHE
ENV E2E_STANDALONE=1 NEXT_TELEMETRY_DISABLED=1 REDIS_URL=redis://redis:6379
RUN pnpm --filter e2e-test-app build
+# Next cannot trace the handler's lazy ioredis require. Keep the locked production closure.
+RUN pnpm --filter e2e-test-app deploy --prod --legacy /runtime-deps
FROM node:22-bookworm-slim
WORKDIR /app
ENV NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 NEXT_TELEMETRY_DISABLED=1
COPY --from=build /app/apps/e2e-test-app/.next/standalone ./
+COPY --from=build /runtime-deps/node_modules ./node_modules
COPY --from=build /app/apps/e2e-test-app/.next/static ./apps/e2e-test-app/.next/static
COPY k8s/e2e/driver.mjs ./apps/e2e-test-app/driver.mjs
USER node
From 96b5f6e31ed9b8727a4828a1fbc3dc7d64a238a1 Mon Sep 17 00:00:00 2001
From: Jason Roy
Date: Thu, 29 Oct 2026 20:48:34 -0700
Subject: [PATCH 3/4] test: build cluster gate against the PR handler after
rc.6 merge
---
.github/workflows/multi-pod-e2e.yml | 7 +------
docs/development.md | 2 +-
k8s/e2e/Dockerfile | 2 +-
3 files changed, 3 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/multi-pod-e2e.yml b/.github/workflows/multi-pod-e2e.yml
index e712c0f..f2556d7 100644
--- a/.github/workflows/multi-pod-e2e.yml
+++ b/.github/workflows/multi-pod-e2e.yml
@@ -21,11 +21,6 @@ jobs:
control: [fixed, reverted]
steps:
- uses: actions/checkout@v4
- # Temporary until #106 lands: build the fixture with the pending rc.6 handler.
- - uses: actions/checkout@v4
- with:
- ref: mrjasonroy/empowering-pumps-ep-59
- path: .handler-under-test
- uses: actions/setup-node@v4
with:
node-version: "22"
@@ -36,7 +31,7 @@ jobs:
run: |
revert=0
if [ "$CONTROL" = reverted ]; then revert=1; fi
- docker build -f k8s/e2e/Dockerfile --build-context handler=.handler-under-test \
+ docker build -f k8s/e2e/Dockerfile \
--build-arg USE_CACHE="$STORAGE" --build-arg REVERT_TIMESTAMP="$revert" \
-t cache-e2e:pr .
- name: Create kind cluster
diff --git a/docs/development.md b/docs/development.md
index 8c9bde3..7d8b609 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -221,7 +221,7 @@ pod logs/events, and Redis tag hash/index dumps, including on expected negative-
To reproduce with Docker, kind, kubectl, Node 22 and OpenSSL installed:
```bash
-docker build -f k8s/e2e/Dockerfile --build-context handler=. \
+docker build -f k8s/e2e/Dockerfile \
--build-arg USE_CACHE=redis -t cache-e2e:pr .
kind create cluster --name cache-e2e
kind load docker-image cache-e2e:pr --name cache-e2e
diff --git a/k8s/e2e/Dockerfile b/k8s/e2e/Dockerfile
index 75f794e..4029b94 100644
--- a/k8s/e2e/Dockerfile
+++ b/k8s/e2e/Dockerfile
@@ -5,7 +5,7 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
COPY packages/cache-handler/package.json packages/cache-handler/package.json
COPY apps/e2e-test-app/package.json apps/e2e-test-app/package.json
RUN pnpm install --frozen-lockfile
-COPY --from=handler packages/cache-handler packages/cache-handler
+COPY packages/cache-handler packages/cache-handler
COPY apps/e2e-test-app apps/e2e-test-app
# Negative control lives only in this throwaway image; never changes the checkout.
ARG REVERT_TIMESTAMP=0
From d2f006e6e4fab2cc3cdcf953b7cf45cb9268dcdb Mon Sep 17 00:00:00 2001
From: Jason Roy
Date: Thu, 29 Oct 2026 20:51:52 -0700
Subject: [PATCH 4/4] test: wait for Service routing before cluster assertions
---
k8s/e2e/driver.mjs | 23 ++++++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
diff --git a/k8s/e2e/driver.mjs b/k8s/e2e/driver.mjs
index a870fd8..03cf2a2 100644
--- a/k8s/e2e/driver.mjs
+++ b/k8s/e2e/driver.mjs
@@ -10,13 +10,17 @@ const secret = process.env.E2E_REVALIDATE_SECRET;
assert.ok(service && secret, "APP_URL and E2E_REVALIDATE_SECRET required");
const redis = new Redis(process.env.REDIS_URL, {
maxRetriesPerRequest: 0,
- retryStrategy: () => null,
+ retryStrategy: (attempt) => (attempt <= 30 ? 100 : null),
});
redis.on("error", (error) => process.stderr.write(`${error.message}\n`));
const ids = Array.from({ length: 19 }, (_, i) => `article-${i}`);
const tags = ids.map((id) => `multi-pod-${id}`);
const state = new Map(ids.map((id) => [id, { version: 1, hold: null, started: false }]));
const server = createServer(async (request, response) => {
+ if (request.url === "/health") {
+ response.end("ready");
+ return;
+ }
const entry = state.get(request.url?.slice(1));
if (!entry) {
response.writeHead(404).end();
@@ -89,6 +93,23 @@ async function freshEverywhere(version, phase) {
let release = () => {};
try {
await new Promise((resolve) => server.listen(8080, "0.0.0.0", resolve));
+ // A ready Redis pod can precede kube-proxy installing its Service routes.
+ await until(
+ () =>
+ redis.ping().then(
+ () => true,
+ () => false,
+ ),
+ "Redis Service reachable",
+ );
+ await until(
+ () =>
+ fetch("http://driver:8080/health", { signal: AbortSignal.timeout(1000) }).then(
+ (response) => response.ok,
+ () => false,
+ ),
+ "origin Service reachable",
+ );
const unauthorized = await fetch(`${service}/api/e2e/multi-pod`, {
method: "POST",
body: JSON.stringify({ tags }),