Skip to content

docs(aws): record grill-me answers #69

docs(aws): record grill-me answers

docs(aws): record grill-me answers #69

name: Integration Tests
on:
# Every branch, not just master: this was master-only from 2026-03 to 2026-08, during which
# the branch accumulated 50 commits of chart changes with no integration run behind them.
# branches: ['**'] rather than a bare `push:` so release tag pushes do not run this workflow
# a second time on top of the branch push that produced them.
#
# No `pull_request:` trigger — every PR branch lives in this repo, so the push event already
# covers it and adding one would double the k3d clusters per commit.
push:
branches: ['**']
workflow_dispatch:
# Integration runs are far more expensive than the unit suite, so superseded runs on the same
# branch are cancelled rather than left to finish.
concurrency:
group: integration-tests-${{ github.ref }}
cancel-in-progress: true
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
jobs:
integration-tests:
name: Integration Tests (${{ matrix.mode.name }})
runs-on: ubuntu-latest
# Without this the job inherits GitHub's 6-hour default. Observed 2026-08-13: a runner whose
# k3d cluster stopped answering kubectl left a job wedged in a log-collection step that ignored
# cancel, force-cancel AND concurrency eviction — all of which need a responsive runner — while
# a queued run sat behind it and a chart stayed half-released. A timeout is the only bound that
# does not depend on the runner cooperating. Sized for the worst legitimate case: scaled installs
# for up to 20m (installTimeout) and then runs `helm test` for up to 10m.
timeout-minutes: 40
strategy:
fail-fast: false
# Five jobs starting at once means five k3d clusters pulling the same large Ontoserver image
# simultaneously, from the shared IP range GitHub-hosted runners use — which registries rate
# limit against. Staggering trades wall-clock for a much smaller burst. Suspected (not proven)
# cause of the 2026-08-13 install stalls; the diagnostics added alongside this will say.
max-parallel: 2
matrix:
mode:
- name: read-only
release: ontoserver-ro
isReadOnly: "true"
expectedHook: ontoserver-ro-ontoserver-test-fhir-ro
unexpectedHook: ontoserver-ro-ontoserver-test-fhir-rw
runHelmTest: true
verifyTraefikRoute: false
valuesFile: ""
needsPostgres: false
needsTraefikCrds: false
needsEnvoyGateway: false
verifyGatewayRoute: false
installTimeout: 10m
- name: read-write
release: ontoserver-rw
isReadOnly: "false"
expectedHook: ontoserver-rw-ontoserver-test-fhir-rw
unexpectedHook: ontoserver-rw-ontoserver-test-fhir-ro
runHelmTest: true
verifyTraefikRoute: false
valuesFile: ""
needsPostgres: false
needsTraefikCrds: false
needsEnvoyGateway: false
verifyGatewayRoute: false
installTimeout: 10m
- name: traefik-https-backend
release: ontoserver-traefik
isReadOnly: "true"
expectedHook: ""
unexpectedHook: ""
runHelmTest: false
verifyTraefikRoute: true
valuesFile: traefik-https-backend-values.yaml
needsPostgres: false
needsTraefikCrds: true
needsEnvoyGateway: false
verifyGatewayRoute: false
installTimeout: 10m
# Gateway API: gateway.yaml and envoy-gateway-policies.yaml were unit-tested only, so
# nothing had ever checked that a real API server accepts them. The Envoy Gateway CRDs
# reject a listener whose protocol and tls block disagree, an HTTPRoute whose
# sectionName matches no listener, or an unknown policy field — none of which
# helm-unittest can see.
#
# Single-instance: this covers the main HTTPRoute plus all three traffic policies, and
# doubles as the negative case for the $closure HTTPRoute, which must render only for
# a scaled StatefulSet.
- name: gateway
release: ontoserver-gw
isReadOnly: "true"
expectedHook: ""
unexpectedHook: ""
runHelmTest: false
verifyTraefikRoute: false
valuesFile: gateway-values.yaml
needsPostgres: false
needsTraefikCrds: false
needsEnvoyGateway: true
verifyGatewayRoute: true
installTimeout: 10m
# Scaled StatefulSet: the only mode that exercises test-closure-scaled-job.yaml,
# i.e. that $closure is pinned to pod-0 and both pods are individually addressable.
# Needs an external PostgreSQL (a scaled deployment cannot use the sidecar DB) and
# its fixture enables a Traefik IngressRoute, hence needsTraefikCrds.
#
# Read-only is mandatory when scaled (enforced by validate-values.yaml): each replica
# has its own Lucene index, so a write served by one replica is missing from the
# others. $closure stays available because it is pinned to pod-0.
- name: scaled
release: ontoserver-scaled
isReadOnly: "true"
expectedHook: ontoserver-scaled-ontoserver-test-closure-scaled
unexpectedHook: ontoserver-scaled-ontoserver-test-fhir-rw
runHelmTest: true
verifyTraefikRoute: false
valuesFile: scaled-values.yaml
needsPostgres: true
needsTraefikCrds: true
needsEnvoyGateway: false
verifyGatewayRoute: false
# Two Ontoserver pods start in parallel but share one runner's CPU.
installTimeout: 20m
steps:
- uses: actions/checkout@v5
- name: Install Helm
uses: azure/setup-helm@v5.0.1
with:
version: v3.18.4
- name: Create k3d cluster
uses: AbsaOSS/k3d-action@v2
with:
cluster-name: ontoserver-test
k3d-version: v5.8.3
# Single node (server only, which k3s leaves schedulable). A second node bought no
# coverage — the chart declares no podAntiAffinity or topologySpreadConstraints, so
# nothing depends on placement, and even scaled's two replicas may co-locate. It did
# cost a second copy of every pulled image on the runner's limited disk.
args: >-
--agents 0
--no-lb
--image rancher/k3s:v1.32.10-k3s1
- name: Wait for Traefik CRDs
if: matrix.mode.needsTraefikCrds
run: |
until kubectl get crd ingressroutes.traefik.io >/dev/null 2>&1; do sleep 2; done
until kubectl wait --for=condition=established crd/ingressroutes.traefik.io --timeout=10s 2>/dev/null; do sleep 2; done
# A scaled deployment must share one database across replicas, so the sidecar DB is
# disabled in scaled-values.yaml and this stands in for the external PostgreSQL that
# ontoserver.config.spring.datasource.url points at.
- name: Deploy PostgreSQL
if: matrix.mode.needsPostgres
run: |
kubectl apply -f - <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
name: postgres
spec:
replicas: 1
selector:
matchLabels:
app: postgres
template:
metadata:
labels:
app: postgres
spec:
containers:
- name: postgres
image: postgres:16
env:
- name: POSTGRES_DB
value: ontoserver
- name: POSTGRES_USER
value: ontoserver
- name: POSTGRES_PASSWORD
value: ontoserver
ports:
- containerPort: 5432
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
memory: 1G
---
apiVersion: v1
kind: Service
metadata:
name: postgres
spec:
selector:
app: postgres
ports:
- port: 5432
targetPort: 5432
EOF
kubectl rollout status deployment/postgres --timeout=3m
# Envoy Gateway's install.yaml bundles the upstream Gateway API CRDs, so this single apply
# provides both the Gateway/HTTPRoute/GatewayClass schemas and the gateway.envoyproxy.io
# policy CRDs plus the controller that programs them. Server-side apply is required: the
# CRDs exceed the 256 KiB annotation limit that client-side apply would need for
# last-applied-configuration.
#
# Pinned rather than tracking latest, like the k3s, k3d and Helm versions above — an
# upstream release should never be able to fail a run that contains no chart change.
#
# --force-conflicts is required, not incidental: k3s installs its own copy of the Gateway
# API CRDs through the bundled Traefik HelmChart, so the apply collides with field manager
# "helm" on .spec.versions and the bundle-version/channel annotations. Without it the
# cluster keeps Traefik's CRD version and Envoy Gateway may be programming against a
# schema it was not built for. Taking ownership pins the CRDs to the version this pinned
# controller ships with. (Traefik's own Gateway API support is unused here — the chart's
# GatewayClass names the Envoy controller, so only Envoy Gateway claims it.)
- name: Install Envoy Gateway
if: matrix.mode.needsEnvoyGateway
run: |
kubectl apply --server-side --force-conflicts -f https://github.com/envoyproxy/gateway/releases/download/v1.8.3/install.yaml
kubectl wait --for=condition=Available --timeout=5m \
-n envoy-gateway-system deployment/envoy-gateway
# The GatewayClass the chart creates references gateway.envoyproxy.io CRDs; wait for
# them to be established so the install below cannot lose a race with the CRD rollout.
for crd in gateways.gateway.networking.k8s.io httproutes.gateway.networking.k8s.io \
clienttrafficpolicies.gateway.envoyproxy.io \
backendtrafficpolicies.gateway.envoyproxy.io \
securitypolicies.gateway.envoyproxy.io \
envoyproxies.gateway.envoyproxy.io; do
kubectl wait --for=condition=established --timeout=2m "crd/$crd"
done
- name: Build chart dependencies
run: helm dependency build ./charts/ontoserver
- name: Install chart
env:
QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }}
run: |
# An array rather than a string: the scaled mode's --set values contain escaped dots
# that word-splitting an unquoted variable would mangle.
EXTRA=()
if [[ -n "${{ matrix.mode.valuesFile }}" ]]; then
EXTRA+=(--values "./charts/ontoserver/tests/fixtures/${{ matrix.mode.valuesFile }}")
fi
if [[ "${{ matrix.mode.needsPostgres }}" == "true" ]]; then
# Password is supplied here rather than in the fixture so the fixture stays
# credential-free; matches the local k3d recipe documented in scaled-values.yaml.
EXTRA+=(--set 'ontoserver.secretConfig.spring\.datasource\.password=ontoserver')
# StatefulSet volumeClaimTemplates need a real storage class; k3s ships local-path.
EXTRA+=(--set ontoserver.deployment.persistence.files.storageClass.provided.enabled=false)
EXTRA+=(--set ontoserver.deployment.persistence.files.storageClass.name=local-path)
fi
helm install ${{ matrix.mode.release }} ./charts/ontoserver \
"${EXTRA[@]}" \
--set ontoserver.deployment.isReadOnly=${{ matrix.mode.isReadOnly }} \
--set ontoserver.managementService.enabled=true \
--set ontoserver.imageCredentials.username="$QUAY_USERNAME" \
--set ontoserver.imageCredentials.password="$QUAY_PASSWORD" \
--set ontoserver.resources.ontoserver.requests.cpu=500m \
--set ontoserver.resources.ontoserver.limits.cpu=2 \
--set ontoserver.resources.ontoserver.requests.memory=2G \
--set ontoserver.resources.ontoserver.limits.memory=2G \
--set ontoserver.resources.ontoserver.initialHeapSize=1500m \
--set ontoserver.resources.ontoserver.maxHeapSize=1500m \
--set ontoserver.resources.db.requests.cpu=250m \
--set ontoserver.resources.db.limits.cpu=1 \
--set ontoserver.resources.db.requests.memory=512Mi \
--set ontoserver.resources.db.limits.memory=1G \
--wait \
--timeout ${{ matrix.mode.installTimeout }}
# `cancelled()` as well as `failure()`: a cancelled run still gets a grace window in which
# always()/cancelled() steps execute, and twice on 2026-08-13 a stalled install was cancelled
# by hand — which is not a failure, so this step did not run and the evidence was lost.
# Every kubectl carries --request-timeout so a wedged API server cannot hang the step that
# exists to explain the wedge, and the step itself is bounded.
- name: Diagnose failed install
if: failure() || cancelled()
timeout-minutes: 3
run: |
echo "=== Node conditions (DiskPressure / MemoryPressure) ==="
kubectl --request-timeout=30s describe node | sed -n '/Conditions:/,/Addresses:/p' || true
echo "=== Runner disk ==="
df -h / || true
docker system df || true
echo "=== Pod status ==="
kubectl --request-timeout=30s get pods -o wide || true
echo "=== Pod description ==="
kubectl --request-timeout=30s describe pods || true
echo "=== Pod logs ==="
kubectl --request-timeout=30s logs -l app=${{ matrix.mode.release }}-ontoserver --all-containers=true --tail=100 || true
echo "=== Events ==="
kubectl --request-timeout=30s get events --sort-by='.lastTimestamp' || true
echo "=== Image pull / eviction events ==="
kubectl --request-timeout=30s get events -o wide \
| grep -Ei 'pull|evict|diskpressure|imagegc|429|throttl' || true
- name: Run integration tests
if: matrix.mode.runHelmTest
run: helm test ${{ matrix.mode.release }} --timeout 10m
- name: Verify mode-specific test hook rendered
if: matrix.mode.runHelmTest
run: |
kubectl get job ${{ matrix.mode.expectedHook }}
if kubectl get job ${{ matrix.mode.unexpectedHook }} >/dev/null 2>&1; then
echo "Unexpected Helm test job found: ${{ matrix.mode.unexpectedHook }}"
exit 1
fi
- name: Verify Traefik route (HTTPS backend)
if: matrix.mode.verifyTraefikRoute
run: |
# Expose Traefik's web entrypoint locally (ports.web.exposedPort=80)
kubectl port-forward -n kube-system svc/traefik 18080:80 &
# Wait for port-forward to be ready (up to 15 s).
# Uses Traefik's built-in /ping health endpoint with --fail so the loop
# only breaks on an actual HTTP 200, not on connection errors returning 0.
for i in $(seq 15); do
curl -sf http://localhost:18080/ping >/dev/null 2>&1 && break
sleep 1
done
# Full path: curl → Traefik (web entrypoint) → ServersTransport (insecureSkipVerify)
# → Ontoserver HTTPS (self-signed /keystore.p12)
curl -sf \
-H "Host: ontoserver.traefik-test.local" \
http://localhost:18080/fhir/metadata \
| grep '"resourceType":"CapabilityStatement"'
- name: Verify Gateway API route (Envoy Gateway)
if: matrix.mode.verifyGatewayRoute
run: |
set -o pipefail
GW=${{ matrix.mode.release }}-gw
# Accepted, plus the per-listener conditions below — deliberately NOT the Gateway-level
# Programmed condition. Envoy Gateway exposes the proxy through a LoadBalancer Service,
# and this cluster is created with --no-lb, so no address is ever assigned and the
# Gateway sits at Programmed=False/AddressNotAssigned forever even when everything the
# chart renders is correct. That condition reports on the cluster's load balancer, not
# on the chart.
kubectl wait --for=condition=Accepted --timeout=5m gateway/"$GW"
# The per-listener conditions are the real assertion a CRDs-only install cannot make:
# Programmed here means the controller translated this listener and pushed it to the
# data plane. A protocol/tls mismatch or a bad certificateRef fails at this level, and
# attachedRoutes proves the HTTPRoute's sectionName actually bound to the listener
# rather than silently matching nothing.
for i in $(seq 45); do
lprog=$(kubectl get gateway "$GW" \
-o jsonpath='{.status.listeners[0].conditions[?(@.type=="Programmed")].status}' 2>/dev/null || true)
lres=$(kubectl get gateway "$GW" \
-o jsonpath='{.status.listeners[0].conditions[?(@.type=="ResolvedRefs")].status}' 2>/dev/null || true)
attached=$(kubectl get gateway "$GW" \
-o jsonpath='{.status.listeners[0].attachedRoutes}' 2>/dev/null || true)
[[ "$lprog" == "True" && "$lres" == "True" && "${attached:-0}" -ge 1 ]] && break
sleep 4
done
echo "listener: Programmed=$lprog ResolvedRefs=$lres attachedRoutes=$attached"
[[ "$lprog" == "True" && "$lres" == "True" && "${attached:-0}" -ge 1 ]] \
|| { kubectl get gateway "$GW" -o yaml; exit 1; }
# HTTPRoute conditions live under status.parents[], per parent Gateway — there is no
# top-level status.conditions, so `kubectl wait --for=condition=Accepted` would hang
# on a route that is already fine. Poll the nested condition instead.
for route in ${{ matrix.mode.release }}-route; do
for i in $(seq 30); do
accepted=$(kubectl get httproute "$route" \
-o jsonpath='{.status.parents[0].conditions[?(@.type=="Accepted")].status}' 2>/dev/null || true)
resolved=$(kubectl get httproute "$route" \
-o jsonpath='{.status.parents[0].conditions[?(@.type=="ResolvedRefs")].status}' 2>/dev/null || true)
[[ "$accepted" == "True" && "$resolved" == "True" ]] && break
sleep 4
done
echo "$route: Accepted=$accepted ResolvedRefs=$resolved"
# ResolvedRefs matters as much as Accepted: it is what fails when backendRefs names
# a Service that does not exist, which is exactly what backendServiceNameOverride
# and the pod0-service default get wrong.
[[ "$accepted" == "True" && "$resolved" == "True" ]] || { kubectl get httproute "$route" -o yaml; exit 1; }
done
# This mode is single-instance, so the $closure route must NOT exist — it renders only
# for a scaled StatefulSet. Guards against the condition being widened by accident.
if kubectl get httproute ${{ matrix.mode.release }}-closure-route >/dev/null 2>&1; then
echo "Unexpected \$closure HTTPRoute on a single-instance deployment"
exit 1
fi
# All three policies must be accepted by the controller, not merely admitted by the
# API server. These use gateway.envoyproxy.io/v1alpha1 and had no coverage at all.
for kind in clienttrafficpolicy/${{ matrix.mode.release }}-client-settings \
backendtrafficpolicy/${{ matrix.mode.release }}-backend-settings \
securitypolicy/${{ matrix.mode.release }}-ip-filter; do
for i in $(seq 30); do
status=$(kubectl get "$kind" \
-o jsonpath='{.status.ancestors[0].conditions[?(@.type=="Accepted")].status}' 2>/dev/null || true)
[[ "$status" == "True" ]] && break
sleep 4
done
echo "$kind: Accepted=$status"
[[ "$status" == "True" ]] || { kubectl get "$kind" -o yaml; exit 1; }
done
# End-to-end, mirroring the Traefik mode: the Envoy fleet is a separate Deployment the
# controller creates in its own namespace, discovered by owning-gateway label rather
# than by a generated name (which carries a hash).
ENVOY_SVC=$(kubectl get svc -n envoy-gateway-system \
-l gateway.envoyproxy.io/owning-gateway-name="$GW" \
-o jsonpath='{.items[0].metadata.name}')
echo "Envoy service: $ENVOY_SVC"
kubectl wait --for=condition=Available --timeout=3m -n envoy-gateway-system \
deployment -l gateway.envoyproxy.io/owning-gateway-name="$GW"
kubectl port-forward -n envoy-gateway-system svc/"$ENVOY_SVC" 18081:80 &
for i in $(seq 30); do
curl -s -o /dev/null -H "Host: ontoserver.gateway-test.local" \
http://localhost:18081/fhir/metadata && break
sleep 2
done
# Full path: curl → Envoy (HTTP listener :80, hostname-matched) → HTTPRoute
# → gw-ontoserver-service → Ontoserver
curl -sf \
-H "Host: ontoserver.gateway-test.local" \
http://localhost:18081/fhir/metadata \
| grep '"resourceType":"CapabilityStatement"'
# always() with no timeout is what wedged for 8 minutes on 2026-08-13 and became
# uncancellable. Every collection step is bounded, and every kubectl gets a request timeout.
- name: Collect test logs
if: always()
timeout-minutes: 3
run: |
echo "=== Events ==="
kubectl --request-timeout=30s get events --sort-by='.lastTimestamp'
echo "=== Ontoserver pod logs (all containers) ==="
kubectl --request-timeout=30s logs -l app=${{ matrix.mode.release }}-ontoserver --all-containers=true --tail=50 || true
echo "=== test-metadata job ==="
kubectl --request-timeout=30s describe job ${{ matrix.mode.release }}-ontoserver-test-metadata || true
echo "=== test-metadata logs ==="
kubectl --request-timeout=30s logs -l job-name=${{ matrix.mode.release }}-ontoserver-test-metadata --tail=200 || true
# Every test hook, not just the mode-specific one: `helm test` fails on ANY hook, so
# collecting only expectedHook leaves the actual failure invisible in the log.
- name: Collect helm test hook logs
if: always() && matrix.mode.runHelmTest
timeout-minutes: 3
run: |
for job in $(kubectl --request-timeout=30s get jobs -o name | sed 's|job.batch/||' | grep -- '-test-'); do
echo "=== job/$job ==="
kubectl --request-timeout=30s get job "$job" -o jsonpath='{.status}' || true
echo ""
echo "--- logs ---"
kubectl --request-timeout=30s logs -l job-name="$job" --tail=200 || true
done
- name: Collect Traefik logs
if: always() && matrix.mode.verifyTraefikRoute
timeout-minutes: 3
run: |
echo "=== Traefik logs ==="
kubectl --request-timeout=30s logs -n kube-system -l app.kubernetes.io/name=traefik --tail=50 || true
- name: Collect Gateway API diagnostics
if: always() && matrix.mode.verifyGatewayRoute
timeout-minutes: 3
run: |
echo "=== GatewayClass / Gateway / HTTPRoute ==="
kubectl --request-timeout=30s get gatewayclass,gateway,httproute -o wide || true
echo "=== Gateway status ==="
kubectl --request-timeout=30s get gateway ${{ matrix.mode.release }}-gw -o yaml || true
echo "=== HTTPRoute status ==="
kubectl --request-timeout=30s get httproute -o yaml || true
echo "=== Envoy Gateway policies ==="
kubectl --request-timeout=30s get clienttrafficpolicy,backendtrafficpolicy,securitypolicy -o yaml || true
echo "=== Envoy Gateway control plane logs ==="
kubectl --request-timeout=30s logs -n envoy-gateway-system deployment/envoy-gateway --tail=100 || true
echo "=== Envoy proxy fleet ==="
kubectl --request-timeout=30s get pods,svc -n envoy-gateway-system -o wide || true
kubectl --request-timeout=30s logs -n envoy-gateway-system \
-l gateway.envoyproxy.io/owning-gateway-name=${{ matrix.mode.release }}-gw \
--tail=100 || true