From 6610905e01c4e3aefe1ce553c66ef72f5ba8449b Mon Sep 17 00:00:00 2001 From: Bugs5382 <12115015+Bugs5382@users.noreply.github.com> Date: Wed, 30 Sep 2026 05:56:04 -0400 Subject: [PATCH] test(e2e): prove cert-manager gets a CryptOS certificate over ACME in kind Stand up a Root and an ACME-serving Intermediate in-process with the e2e harness (software keys, embedded etcd, no TPM) and let cert-manager in a kind cluster enrol against it with an External Account Binding and an http-01 solver behind Contour. The test checks that the Certificate goes Ready, the chain verifies to the CryptOS root, the SANs are the requested name, the node recorded the serial, the ingress serves it, and a forced renewal yields a new, recorded serial. The handler the test drives is the one the boot serves: the ACME wiring in the boot moves into cinit.NewACMEHandler, and the issued-set recorder is exported as cinit.IssuedRecorder, with no change in behaviour. task e2e:kind runs it on a Linux host with docker and skips without one. The ci-kind-acme workflow runs it on pull requests that touch the ACME path. kind, kubectl, the node image, cert-manager, Contour and the test app are pinned by checksum or digest in test/kind/versions.env. Signed-off-by: Bugs5382 <12115015+Bugs5382@users.noreply.github.com> --- .github/workflows/ci-kind-acme.yml | 54 +++ README.md | 6 + Taskfile.yml | 5 + internal/e2e/kind_acme_e2e_test.go | 670 +++++++++++++++++++++++++++++ internal/init/acme.go | 26 ++ internal/init/renew_live_test.go | 2 +- internal/init/revocation.go | 4 +- internal/init/revocation_test.go | 2 +- internal/init/run.go | 18 +- test/kind/kind-config.yaml | 15 + test/kind/run.sh | 175 ++++++++ test/kind/versions.env | 27 ++ test/kind/whoami.yaml | 60 +++ 13 files changed, 1046 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/ci-kind-acme.yml create mode 100644 internal/e2e/kind_acme_e2e_test.go create mode 100644 test/kind/kind-config.yaml create mode 100755 test/kind/run.sh create mode 100644 test/kind/versions.env create mode 100644 test/kind/whoami.yaml diff --git a/.github/workflows/ci-kind-acme.yml b/.github/workflows/ci-kind-acme.yml new file mode 100644 index 0000000..cf2f4af --- /dev/null +++ b/.github/workflows/ci-kind-acme.yml @@ -0,0 +1,54 @@ +name: ACME end-to-end (cert-manager in kind) + +# Proves a real Kubernetes client gets a real certificate from CryptOS over +# ACME: cert-manager in a kind cluster enrols (http-01, through Contour) with +# an in-process Intermediate on the runner, and the test checks the chain, the +# SANs, the node's issued set and a forced renewal. See test/kind/run.sh. +# +# Pull requests only, and only when the ACME path or this harness changes. It +# is not a required check. Tools, images and manifests are pinned with +# checksums or digests in test/kind/versions.env. +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + paths: + - "internal/acme/**" + - "internal/config/**" + - "internal/node/**" + - "internal/e2e/**" + - "internal/init/acme.go" + - "test/kind/**" + - ".github/workflows/ci-kind-acme.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + kind-acme: + name: cert-manager gets a CryptOS certificate over ACME + if: github.event.pull_request.draft != true + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Cache kind, kubectl and manifests + uses: actions/cache@v4 + with: + path: ~/.cache/cryptos-e2e-kind + key: kind-acme-${{ runner.arch }}-${{ hashFiles('test/kind/versions.env') }} + + - name: Run the kind ACME end-to-end test + run: bash test/kind/run.sh diff --git a/README.md b/README.md index 1055382..029dd8d 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ internal/ config/ # machine config parser + validator bootstrap/ # bootstrap admin cert loading + first-ceremony rotation build/ # kernel config, UKI assembly recipes, SquashFS templates +test/kind/ # kind + cert-manager ACME end-to-end harness (task e2e:kind) testdata/configs/ # sample machine configs ``` @@ -51,8 +52,11 @@ Requires Go 1.24+, [`go-task`](https://taskfile.dev), `golangci-lint`, `golic`, task ci # fmt + lint + vet + test + build (both binaries) task build # produces bin/init and bin/cryptosctl, stamped with the build identity task license # re-inject Apache 2.0 headers via golic +task e2e:kind # Linux + docker: cert-manager in kind gets a certificate over ACME ``` +`task e2e:kind` builds a Root and an ACME-serving Intermediate in-process (software keys, no TPM), stands up a kind cluster with cert-manager and Contour, and checks that a `Certificate` goes Ready with a chain to the root and renews to a new serial. It downloads pinned, checksum-checked kind, kubectl and manifests ([`test/kind/versions.env`](test/kind/versions.env)), needs sudo once to add a `/etc/hosts` line for the test name, and skips when docker isn't available. + `bin/cryptosctl version` prints the version, commit, and build date the binary was built from (`git describe --tags --always --dirty`; see [`build/README.md`](build/README.md#build-identity)); pass `--endpoint` to also show the node's version. The image pipeline (`task image` — hardened kernel build, SquashFS rootfs, UKI assembly + Secure Boot signing) has draft recipes under `build/` that run on a Linux build host; see [`build/README.md`](build/README.md). They are written but not yet executed end to end. The QEMU + `swtpm` integration harness lands in a subsequent PR. @@ -90,6 +94,8 @@ GitHub Actions: - **`ci-go`** ([`ci-go.yml`](.github/workflows/ci-go.yml)) — `task ci` (format, lint, vet, test, build) on every pull request + push to `main`, on a GitHub-hosted Linux runner. Draft pull requests are skipped; CI runs when the PR is marked ready. - **`ci-image`** ([`ci-image.yml`](.github/workflows/ci-image.yml)) — builds the UKI on a **GitHub-hosted runner** (amd64 on `ubuntu-latest`, arm64 on `ubuntu-24.04-arm`), installing the kernel / `ukify` / `sbsign` toolchain per run. Runs on push to `main`, tags, and manual dispatch; use `workflow_dispatch` on a branch to validate image changes before merging. On `main` it signs with a per-run ephemeral key as a smoke test and uploads nothing. On a `v*` tag it builds the unsigned [release assets](#release-assets) and attaches them to the tag's release (a draft, marked pre-release for `-alpha`/`-beta`/`-rc` tags, if none exists yet); dispatch with `release_assets` builds them without publishing. +- **`ci-kind-acme`** ([`ci-kind-acme.yml`](.github/workflows/ci-kind-acme.yml)) — `test/kind/run.sh` (the `task e2e:kind` harness) on pull requests that touch the ACME, config, node or e2e code, on a GitHub-hosted runner. Drafts are skipped, and it isn't a required check. + The QEMU + `swtpm` integration boot is run on a real host by the operator, not in CI. ## 🔑 Management surfaces diff --git a/Taskfile.yml b/Taskfile.yml index 0590319..f3aa52a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -179,6 +179,11 @@ tasks: cmds: - bash build/ci/qemu-run.sh {{.ARCH | default "amd64"}} + e2e:kind: + desc: "[Linux] cert-manager in a kind cluster enrols over ACME with an in-process Intermediate (needs docker; skips without it)" + cmds: + - bash test/kind/run.sh + test:integration: desc: "[Linux] Phase 1 QEMU + swtpm end-to-end acceptance (needs the toolchain; see test/integration)" cmds: diff --git a/internal/e2e/kind_acme_e2e_test.go b/internal/e2e/kind_acme_e2e_test.go new file mode 100644 index 0000000..9f07c19 --- /dev/null +++ b/internal/e2e/kind_acme_e2e_test.go @@ -0,0 +1,670 @@ +package e2e_test + +/* +Apache License 2.0 + +Copyright 2026 Shane + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file proves that a real Kubernetes client gets a real certificate from +// CryptOS over ACME. The node side is the same in-process hierarchy as +// hierarchy_e2e_test.go (software key backend, embedded etcd, no TPM): a Root +// signs an Intermediate, and the Intermediate serves ACME from a pki.acme +// block parsed by config.Parse and built by cinit.NewACMEHandler, which is +// what the boot serves. The client side is cert-manager in a kind cluster, +// enrolling through an http-01 solver behind the cluster's ingress. +// +// The cluster is stood up by test/kind/run.sh (task e2e:kind), which sets the +// CRYPTOS_E2E_KIND_* variables below. Without them the test skips. + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "strings" + "testing" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" + + "github.com/CryptOS-PKI/cryptos/internal/acme" + "github.com/CryptOS-PKI/cryptos/internal/config" + cinit "github.com/CryptOS-PKI/cryptos/internal/init" + "github.com/CryptOS-PKI/cryptos/internal/node" + "github.com/CryptOS-PKI/cryptos/internal/revocation" + "github.com/CryptOS-PKI/cryptos/internal/storage/etcd" +) + +const ( + kindNamespace = "cryptos-e2e" + kindIssuer = "cryptos-acme" + kindCertificate = "whoami" + kindSecret = "whoami-tls" + kindEABSecret = "cryptos-acme-eab" + kindEABKeyID = "kind-e2e" + kindLeafProfile = "acme-leaf" + + // acmeOriginAddr is the plain-HTTP origin the ACME handler is served on, + // loopback only; acmeFrontPort is the TLS front the cluster dials. + acmeOriginAddr = "127.0.0.1:8555" + acmeFrontPort = "8443" + + certReadyTimeout = 5 * time.Minute +) + +// kindEnv is what test/kind/run.sh hands the test about the cluster it built. +type kindEnv struct { + // hostIP is the host's address on the kind docker network: where pods + // reach the ACME front, and the IP SAN on the front's certificate. + hostIP string + // hostname is the name the Certificate requests. It resolves to the + // cluster's ingress both on the host (so the node's http-01 fetch lands + // there) and inside the cluster (so cert-manager's self-check does). + hostname string + kubeconfig string + kubectl string +} + +func kindEnvOrSkip(t *testing.T) kindEnv { + t.Helper() + if os.Getenv("CRYPTOS_E2E_KIND") != "1" { + t.Skip("kind ACME e2e: not requested; run `task e2e:kind` on a Linux host with docker") + } + env := kindEnv{ + hostIP: os.Getenv("CRYPTOS_E2E_KIND_HOST_IP"), + hostname: os.Getenv("CRYPTOS_E2E_KIND_HOSTNAME"), + kubeconfig: os.Getenv("CRYPTOS_E2E_KIND_KUBECONFIG"), + kubectl: os.Getenv("CRYPTOS_E2E_KIND_KUBECTL"), + } + if env.kubectl == "" { + env.kubectl = "kubectl" + } + if net.ParseIP(env.hostIP) == nil || env.hostname == "" || env.kubeconfig == "" { + t.Fatalf("kind ACME e2e: CRYPTOS_E2E_KIND_HOST_IP, _HOSTNAME and _KUBECONFIG must be set (got %q, %q, %q)", + env.hostIP, env.hostname, env.kubeconfig) + } + return env +} + +// TestKindCertManagerACME drives cert-manager against an ACME-serving +// Intermediate: the Certificate goes Ready with a chain to the CryptOS root, +// the node records the issuance, the ingress serves it, and a forced renewal +// yields a new serial that is recorded too. +func TestKindCertManagerACME(t *testing.T) { + env := kindEnvOrSkip(t) + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + t.Cleanup(cancel) + + n := startACMENode(t, ctx, env) + rootCert, rootPEM, revStore := n.root, n.rootPEM, n.revStore + + kc := kubectlRunner{t: t, env: env} + applyIssuerAndCertificate(t, ctx, kc, rootPEM, n.baseURL, n.eabKeyB64, env.hostname) + + first := waitForIssuedCertificate(t, ctx, kc, "", certReadyTimeout) + assertCertificate(t, ctx, first, rootCert, env.hostname, revStore) + assertServedByIngress(t, ctx, env.hostname, rootCert, first.leaf.SerialNumber.Text(16)) + + // Force a renewal the way `cmctl renew` does: set Issuing=True on the + // Certificate's status, which cert-manager's trigger controller acts on. + firstSerial := first.leaf.SerialNumber.Text(16) + t.Logf("forcing renewal of %s/%s (current serial %s)", kindNamespace, kindCertificate, firstSerial) + patch := fmt.Sprintf(`[{"op":"add","path":"/status/conditions/-","value":{"type":"Issuing","status":"True","reason":"ManuallyTriggered","message":"Certificate re-issuance manually triggered by the CryptOS kind e2e test","lastTransitionTime":%q}}]`, + time.Now().UTC().Format(time.RFC3339)) + if _, err := kc.run(ctx, "", "-n", kindNamespace, "patch", "certificate", kindCertificate, + "--subresource=status", "--type=json", "-p", patch); err != nil { + t.Fatalf("trigger renewal: %v", err) + } + + second := waitForIssuedCertificate(t, ctx, kc, firstSerial, certReadyTimeout) + secondSerial := second.leaf.SerialNumber.Text(16) + if secondSerial == firstSerial { + t.Fatalf("renewal kept serial %s, want a new one", firstSerial) + } + t.Logf("renewed: serial %s -> %s", firstSerial, secondSerial) + assertCertificate(t, ctx, second, rootCert, env.hostname, revStore) + assertServedByIngress(t, ctx, env.hostname, rootCert, secondSerial) +} + +// acmeNode is the ACME-serving Intermediate the cluster enrols against. +type acmeNode struct { + root *x509.Certificate + rootPEM []byte + revStore *revocation.Store + baseURL string + eabKeyB64 string +} + +// startACMENode builds a Root, an Intermediate it signs, and serves ACME from +// the Intermediate behind a TLS front on env.hostIP. +func startACMENode(t *testing.T, ctx context.Context, env kindEnv) acmeNode { + t.Helper() + // Root, then an Intermediate it signs, exactly as TestHierarchyE2E does. + rootStore := newStore(t) + establishRoot(t, ctx, rootStore) + rootID, err := rootStore.Identity(ctx) + if err != nil { + t.Fatalf("root Identity: %v", err) + } + rootCert, err := x509.ParseCertificate(rootID.ChainDer[0]) + if err != nil { + t.Fatalf("parse root cert: %v", err) + } + rootPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootCert.Raw}) + t.Logf("root established: subject=%q serial=%s", rootCert.Subject, rootCert.SerialNumber.Text(16)) + + subStore, cli := newStoreWithClient(t) + subKey := newP384Key(t) + subCSR := buildCSR(t, subKey, pkix.Name{CommonName: "ACME Issuing G1"}) + if err := subStore.StageSubordinate(ctx, subCSR, marshalECKey(t, subKey), marshalECPub(t, subKey)); err != nil { + t.Fatalf("StageSubordinate: %v", err) + } + chainDER, _, _, err := newRootCASigner(t, ctx, rootStore, rootCert, rootProfilesConfig()).SignSubordinate(ctx, subCSR, "sub-ca") + if err != nil { + t.Fatalf("root SignSubordinate: %v", err) + } + + eabKey := make([]byte, config.MinEABKeyBytes) + if _, err := rand.Read(eabKey); err != nil { + t.Fatalf("EAB key: %v", err) + } + eabKeyB64 := base64.RawURLEncoding.EncodeToString(eabKey) + baseURL := "https://" + net.JoinHostPort(env.hostIP, acmeFrontPort) + "/acme" + subCfg := intermediateACMEConfig(t, string(rootPEM), env, baseURL, eabKeyB64) + + parentTrust, err := subCfg.ParentTrust() + if err != nil { + t.Fatalf("ParentTrust: %v", err) + } + enroller, err := node.NewSubordinateEnroller(subStore, parentTrust) + if err != nil { + t.Fatalf("NewSubordinateEnroller: %v", err) + } + if _, err := enroller.AcceptCertificate(ctx, chainDER); err != nil { + t.Fatalf("AcceptCertificate: %v", err) + } + subCert, err := x509.ParseCertificate(chainDER[0]) + if err != nil { + t.Fatalf("parse intermediate: %v", err) + } + t.Logf("intermediate committed: subject=%q serial=%s", subCert.Subject, subCert.SerialNumber.Text(16)) + + // The boot wires the issued-set recorder onto the signer before any + // listener starts; do the same so ACME issuance lands in ListIssued. + revStore := revocation.NewStore(cli) + signer := newSubordinateCASigner(t, ctx, subStore, subCert, subCfg).WithRecorder(cinit.IssuedRecorder(revStore)) + + startACME(t, ctx, cli, signer, subCfg, env) + return acmeNode{root: rootCert, rootPEM: rootPEM, revStore: revStore, baseURL: baseURL, eabKeyB64: eabKeyB64} + +} + +// newStoreWithClient is newStore that also hands back the etcd client, which +// the ACME and revocation stores are built on in production. +func newStoreWithClient(t *testing.T) (*node.Store, *clientv3.Client) { + t.Helper() + srv, err := etcd.Open(t.TempDir()) + if err != nil { + t.Fatalf("etcd.Open: %v", err) + } + t.Cleanup(func() { _ = srv.Close() }) + cli, err := srv.Client() + if err != nil { + t.Fatalf("etcd.Client: %v", err) + } + t.Cleanup(func() { _ = cli.Close() }) + s, err := node.New(cli) + if err != nil { + t.Fatalf("node.New: %v", err) + } + return s, cli +} + +// intermediateACMEConfig is the Intermediate's machine config with ACME on: +// a leaf profile ACME issues under, a server profile for the TLS front that +// carries the host IP as its SAN, and the pki.acme block with one External +// Account Binding key. It goes through config.Parse so validateACME runs. +func intermediateACMEConfig(t *testing.T, rootPEM string, env kindEnv, baseURL, eabKeyB64 string) *config.Config { + t.Helper() + adminFP := newFP(t) + suffix := env.hostname[strings.Index(env.hostname, ".")+1:] + yaml := fmt.Sprintf(`apiVersion: cryptos.dev/v1alpha1 +kind: MachineConfig +metadata: + name: issuing-kind +role: + kind: intermediate +network: + interface: eth0 + address: 10.0.0.11/24 + gateway: 10.0.0.1 +bootstrap: + admin_cert_sha256: "%s" +pki: + root_key_alg: ECDSA-P384 + root_subject: + common_name: "ACME Issuing G1" + organization: "ACME" + country: "US" + root_validity_years: 10 + path_len_constraint: 0 + parent: + ca_cert_pem: | +%s + profiles: + - name: %s + key_alg: ECDSA-P384 + subject: + common_name: acme + validity_days: 90 + basic_constraints: + is_ca: false + key_usage: [digital_signature] + ext_key_usage: [server_auth] + - name: acme-front + key_alg: ECDSA-P384 + subject: + common_name: cryptos-acme-front + validity_days: 7 + basic_constraints: + is_ca: false + key_usage: [digital_signature] + ext_key_usage: [server_auth] + sans: + ip: ["%s"] + acme: + base_url: %s + profile: %s + allowed_identifier_suffixes: [%s] + external_account_keys: + - key_id: %s + hmac_key_base64: %s +`, hex.EncodeToString(adminFP[:]), indentPEM(rootPEM, " "), kindLeafProfile, + env.hostIP, baseURL, kindLeafProfile, suffix, kindEABKeyID, eabKeyB64) + cfg, err := config.Parse([]byte(yaml)) + if err != nil { + t.Fatalf("config.Parse (intermediate with pki.acme): %v", err) + } + return cfg +} + +// startACME serves the node's ACME handler on a loopback origin and puts a +// TLS front in front of it on the kind network, the deployment docs/acme.md +// describes. The front's certificate is issued by the Intermediate, so the +// cluster trusts it through the same root it is handed as caBundle. +func startACME(t *testing.T, ctx context.Context, cli *clientv3.Client, signer *node.CASigner, cfg *config.Config, env kindEnv) { + t.Helper() + + h, err := cinit.NewACMEHandler(cli, signer, nil, cfg.PKI.ACME) + if err != nil { + t.Fatalf("NewACMEHandler: %v", err) + } + stop, err := acme.Serve(ctx, acmeOriginAddr, h) + if err != nil { + t.Fatalf("acme.Serve: %v", err) + } + t.Cleanup(func() { + sctx, scancel := context.WithTimeout(context.Background(), 5*time.Second) + defer scancel() + _ = stop(sctx) + }) + t.Logf("ACME origin up on http://%s (base_url %s)", acmeOriginAddr, cfg.PKI.ACME.BaseURL) + + frontKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + t.Fatalf("front key: %v", err) + } + frontChain, _, err := signer.IssueLeafForNames(ctx, buildCSR(t, frontKey, pkix.Name{CommonName: "cryptos-acme-front"}), "acme-front", nil) + if err != nil { + t.Fatalf("issue the TLS front certificate: %v", err) + } + front := tls.Certificate{Certificate: frontChain, PrivateKey: frontKey} + + origin := &url.URL{Scheme: "http", Host: acmeOriginAddr} + proxy := httputil.NewSingleHostReverseProxy(origin) + lis, err := net.Listen("tcp", net.JoinHostPort(env.hostIP, acmeFrontPort)) + if err != nil { + t.Fatalf("listen for the ACME TLS front: %v", err) + } + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Logf("ACME front: %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr) + proxy.ServeHTTP(w, r) + }), + TLSConfig: &tls.Config{Certificates: []tls.Certificate{front}, MinVersion: tls.VersionTLS12}, + ReadHeaderTimeout: 5 * time.Second, + } + go func() { _ = srv.ServeTLS(lis, "", "") }() + t.Cleanup(func() { _ = srv.Close() }) + t.Logf("ACME TLS front up on https://%s", lis.Addr()) +} + +// applyIssuerAndCertificate creates the EAB secret, the ClusterIssuer that +// points cert-manager at the node, and the Certificate for hostname. +func applyIssuerAndCertificate(t *testing.T, ctx context.Context, kc kubectlRunner, rootPEM []byte, baseURL, eabKeyB64, hostname string) { + t.Helper() + manifest := fmt.Sprintf(`apiVersion: v1 +kind: Secret +metadata: + name: %[1]s + namespace: cert-manager +type: Opaque +stringData: + hmac: %[2]s +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: %[3]s +spec: + acme: + server: %[4]s/directory + caBundle: %[5]s + privateKeySecretRef: + name: %[3]s-account + externalAccountBinding: + keyID: %[6]s + keySecretRef: + name: %[1]s + key: hmac + solvers: + - http01: + ingress: + ingressClassName: contour +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: %[7]s + namespace: %[8]s +spec: + secretName: %[9]s + dnsNames: [%[10]s] + privateKey: + algorithm: ECDSA + size: 384 + issuerRef: + kind: ClusterIssuer + name: %[3]s +`, kindEABSecret, eabKeyB64, kindIssuer, baseURL, base64.StdEncoding.EncodeToString(rootPEM), + kindEABKeyID, kindCertificate, kindNamespace, kindSecret, hostname) + // The cert-manager webhook can report Available a few seconds before it + // admits requests, so a refused apply right after install is retried. + deadline := time.Now().Add(90 * time.Second) + for { + _, err := kc.run(ctx, manifest, "apply", "-f", "-") + if err == nil { + break + } + if time.Now().After(deadline) { + t.Fatalf("apply ClusterIssuer and Certificate: %v", err) + } + t.Logf("apply refused, retrying: %v", err) + time.Sleep(5 * time.Second) + } + t.Logf("applied ClusterIssuer %s (server %s/directory) and Certificate %s/%s for %s", + kindIssuer, baseURL, kindNamespace, kindCertificate, hostname) + t.Cleanup(func() { + cctx, ccancel := context.WithTimeout(context.Background(), 30*time.Second) + defer ccancel() + _, _ = kc.run(cctx, manifest, "delete", "--ignore-not-found", "--wait=false", "-f", "-") + }) +} + +// issuedCertificate is what cert-manager wrote into the Certificate's secret. +type issuedCertificate struct { + leaf *x509.Certificate + intermediates []*x509.Certificate + key any +} + +// waitForIssuedCertificate polls until the Certificate is Ready and its secret +// holds a leaf whose serial is not notSerial, dumping cert-manager's view of +// the enrolment if that does not happen in time. +func waitForIssuedCertificate(t *testing.T, ctx context.Context, kc kubectlRunner, notSerial string, timeout time.Duration) issuedCertificate { + t.Helper() + deadline := time.Now().Add(timeout) + lastState := "" + for { + ready, _ := kc.run(ctx, "", "-n", kindNamespace, "get", "certificate", kindCertificate, + "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].status}/{.status.conditions[?(@.type=="Ready")].reason}`) + if ready != lastState { + t.Logf("Certificate %s/%s Ready=%s", kindNamespace, kindCertificate, ready) + lastState = ready + } + if strings.HasPrefix(ready, "True/") { + got, err := readIssuedCertificate(ctx, kc) + if err != nil { + t.Logf("Certificate Ready but secret not readable yet: %v", err) + } else if got.leaf.SerialNumber.Text(16) != notSerial { + return got + } + } + if time.Now().After(deadline) { + kc.dumpDiagnostics(ctx) + t.Fatalf("Certificate %s/%s did not get a new certificate within %s (last Ready=%q)", + kindNamespace, kindCertificate, timeout, ready) + } + select { + case <-ctx.Done(): + kc.dumpDiagnostics(context.Background()) + t.Fatalf("context done waiting for the Certificate: %v", ctx.Err()) + case <-time.After(3 * time.Second): + } + } +} + +func readIssuedCertificate(ctx context.Context, kc kubectlRunner) (issuedCertificate, error) { + var out issuedCertificate + crtB64, err := kc.run(ctx, "", "-n", kindNamespace, "get", "secret", kindSecret, "-o", `jsonpath={.data.tls\.crt}`) + if err != nil { + return out, err + } + keyB64, err := kc.run(ctx, "", "-n", kindNamespace, "get", "secret", kindSecret, "-o", `jsonpath={.data.tls\.key}`) + if err != nil { + return out, err + } + crtPEM, err := base64.StdEncoding.DecodeString(strings.TrimSpace(crtB64)) + if err != nil { + return out, fmt.Errorf("decode tls.crt: %w", err) + } + keyPEM, err := base64.StdEncoding.DecodeString(strings.TrimSpace(keyB64)) + if err != nil { + return out, fmt.Errorf("decode tls.key: %w", err) + } + var certs []*x509.Certificate + for rest := crtPEM; ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return out, fmt.Errorf("parse tls.crt: %w", err) + } + certs = append(certs, c) + } + if len(certs) == 0 { + return out, errors.New("tls.crt holds no certificate") + } + block, _ := pem.Decode(keyPEM) + if block == nil { + return out, errors.New("tls.key holds no PEM block") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + if ec, ecErr := x509.ParseECPrivateKey(block.Bytes); ecErr == nil { + key, err = ec, nil + } + } + if err != nil { + return out, fmt.Errorf("parse tls.key: %w", err) + } + out.leaf, out.intermediates, out.key = certs[0], certs[1:], key + return out, nil +} + +// assertCertificate checks what a relying party and an auditor would: the +// chain verifies to the CryptOS root for hostname, the SANs are exactly the +// requested name, the key is the P-384 key cert-manager generated, and the +// node's issued set holds the serial under the ACME profile. +func assertCertificate(t *testing.T, ctx context.Context, got issuedCertificate, root *x509.Certificate, hostname string, revStore *revocation.Store) { + t.Helper() + serial := got.leaf.SerialNumber.Text(16) + + roots := x509.NewCertPool() + roots.AddCert(root) + inter := x509.NewCertPool() + for _, c := range got.intermediates { + inter.AddCert(c) + } + chains, err := got.leaf.Verify(x509.VerifyOptions{ + DNSName: hostname, + Roots: roots, + Intermediates: inter, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + if err != nil { + t.Fatalf("secret %s chain does not verify to the CryptOS root: %v", kindSecret, err) + } + t.Logf("serial %s verifies: %d-certificate path to %q", serial, len(chains[0]), root.Subject) + + if len(got.leaf.DNSNames) != 1 || got.leaf.DNSNames[0] != hostname { + t.Fatalf("leaf DNS SANs = %v, want [%s]", got.leaf.DNSNames, hostname) + } + if len(got.leaf.IPAddresses) != 0 || len(got.leaf.EmailAddresses) != 0 || len(got.leaf.URIs) != 0 { + t.Fatalf("leaf carries non-DNS SANs: ip=%v email=%v uri=%v", got.leaf.IPAddresses, got.leaf.EmailAddresses, got.leaf.URIs) + } + pub, ok := got.leaf.PublicKey.(*ecdsa.PublicKey) + if !ok || pub.Curve != elliptic.P384() { + t.Fatalf("leaf key is %T, want ECDSA P-384", got.leaf.PublicKey) + } + priv, ok := got.key.(*ecdsa.PrivateKey) + if !ok || !priv.PublicKey.Equal(pub) { + t.Fatal("tls.key does not match the leaf's public key") + } + + rec, found, err := revStore.GetIssued(ctx, serial) + if err != nil { + t.Fatalf("GetIssued(%s): %v", serial, err) + } + if !found { + all, _ := revStore.ListIssued(ctx) + t.Fatalf("serial %s is not in the node's issued set (%d records)", serial, len(all)) + } + if rec.ProfileName != kindLeafProfile || !bytes.Equal(rec.DER, got.leaf.Raw) { + t.Fatalf("issued record for %s: profile=%q der-match=%v, want profile %q and the served DER", + serial, rec.ProfileName, bytes.Equal(rec.DER, got.leaf.Raw), kindLeafProfile) + } + t.Logf("node recorded serial %s under profile %q", serial, rec.ProfileName) +} + +// assertServedByIngress fetches https://hostname through the cluster ingress +// with only the CryptOS root trusted, and waits until the ingress presents the +// certificate with wantSerial. +func assertServedByIngress(t *testing.T, ctx context.Context, hostname string, root *x509.Certificate, wantSerial string) { + t.Helper() + roots := x509.NewCertPool() + roots.AddCert(root) + client := &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, ServerName: hostname, MinVersion: tls.VersionTLS12}, + }, + } + deadline := time.Now().Add(2 * time.Minute) + var last string + for { + resp, err := client.Get("https://" + hostname + "/") + if err == nil { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + _ = resp.Body.Close() + served := resp.TLS.PeerCertificates[0].SerialNumber.Text(16) + if resp.StatusCode == http.StatusOK && served == wantSerial { + t.Logf("ingress serves serial %s for %s (HTTP %d, %d body bytes)", served, hostname, resp.StatusCode, len(body)) + return + } + last = fmt.Sprintf("HTTP %d with serial %s", resp.StatusCode, served) + } else { + last = err.Error() + } + if time.Now().After(deadline) { + t.Fatalf("ingress never served serial %s for %s; last: %s", wantSerial, hostname, last) + } + select { + case <-ctx.Done(): + t.Fatalf("context done waiting for the ingress: %v", ctx.Err()) + case <-time.After(3 * time.Second): + } + } +} + +// kubectlRunner runs kubectl against the kind cluster's kubeconfig. +type kubectlRunner struct { + t *testing.T + env kindEnv +} + +func (k kubectlRunner) run(ctx context.Context, stdin string, args ...string) (string, error) { + cctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + full := append([]string{"--kubeconfig", k.env.kubeconfig}, args...) + cmd := exec.CommandContext(cctx, k.env.kubectl, full...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + return stdout.String(), fmt.Errorf("kubectl %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return stdout.String(), nil +} + +// dumpDiagnostics logs cert-manager's view of the enrolment so a red run says +// which step stalled without a rerun. +func (k kubectlRunner) dumpDiagnostics(ctx context.Context) { + k.t.Helper() + for _, args := range [][]string{ + {"-n", kindNamespace, "describe", "certificate,certificaterequest,order,challenge"}, + {"describe", "clusterissuer", kindIssuer}, + {"-n", kindNamespace, "get", "ingress,pods,svc", "-o", "wide"}, + {"-n", "cert-manager", "logs", "deploy/cert-manager", "--tail=200"}, + } { + out, err := k.run(ctx, "", args...) + k.t.Logf("--- kubectl %s ---\n%s", strings.Join(args, " "), out) + if err != nil { + k.t.Logf("(%v)", err) + } + } +} diff --git a/internal/init/acme.go b/internal/init/acme.go index bfd9855..efe87f7 100644 --- a/internal/init/acme.go +++ b/internal/init/acme.go @@ -26,6 +26,8 @@ import ( "log" "time" + clientv3 "go.etcd.io/etcd/client/v3" + "github.com/CryptOS-PKI/cryptos/internal/acme" "github.com/CryptOS-PKI/cryptos/internal/config" "github.com/CryptOS-PKI/cryptos/internal/node" @@ -38,6 +40,30 @@ import ( // origin behind it. const defaultACMEHTTPPort = 8555 +// NewACMEHandler builds the ACME handler a node serves for its validated +// pki.acme block: ACME state in the node's etcd, issuance through signer under +// the block's profile, and the RFC 8555 http-01 validator. The boot serves +// exactly this handler, and the kind end-to-end test drives the same one, so +// what that test proves is what a node runs. revoke may be nil, in which case +// revoke-cert is not offered. +func NewACMEHandler(cli *clientv3.Client, signer *node.CASigner, revoke acme.RevokeFunc, c *config.ACME) (*acme.Handler, error) { + opts, err := acmeOptions(c) + if err != nil { + return nil, err + } + h, err := acme.NewHandler( + acme.NewStore(cli), + acmeIssuer(signer, c.Profile), + revoke, + acme.HTTP01Validator(0), + opts, + ) + if err != nil { + return nil, fmt.Errorf("init: build the ACME handler: %w", err) + } + return h, nil +} + // acmeIssuer returns the acme.IssueFunc backed by this node's CA signer. // // It routes through IssueLeafForNames so the certificate's SANs are the diff --git a/internal/init/renew_live_test.go b/internal/init/renew_live_test.go index fb94b1f..48d20ee 100644 --- a/internal/init/renew_live_test.go +++ b/internal/init/renew_live_test.go @@ -133,7 +133,7 @@ func TestRecertifiedIssuerIsLiveEverywhere(t *testing.T) { BasicConstraints: config.BasicConstraints{IsCA: true, PathLen: &pathLen}}, }}} signer := node.NewCASigner(load, issuer, func(context.Context) (*config.Config, error) { return cfg, nil }). - WithRecorder(issuedRecorder(revStore)) + WithRecorder(IssuedRecorder(revStore)) revoker := &nodeRevoker{store: revStore, crlBuilder: revocation.NewCRLBuilder(revStore, time.Hour), load: load, issuer: issuer} responder := newOCSPResponder(store, load, issuer, 0) ocspFn := revoker.ocspFn(revocation.NewOCSPResponder(revStore), responder) diff --git a/internal/init/revocation.go b/internal/init/revocation.go index c7b7193..ebfe973 100644 --- a/internal/init/revocation.go +++ b/internal/init/revocation.go @@ -49,11 +49,11 @@ func nonzero(v, fallback uint32) uint32 { return v } -// issuedRecorder returns a CASigner recorder that parses a freshly minted +// IssuedRecorder returns a CASigner recorder that parses a freshly minted // certificate and persists it into the revocation issued set. The recorder is // called after a successful Sign but before the certificate is returned to the // caller, so the issued set never drifts from what a caller received. -func issuedRecorder(store *revocation.Store) func(ctx context.Context, der []byte, profileName string) error { +func IssuedRecorder(store *revocation.Store) func(ctx context.Context, der []byte, profileName string) error { return func(ctx context.Context, der []byte, profileName string) error { cert, err := x509.ParseCertificate(der) if err != nil { diff --git a/internal/init/revocation_test.go b/internal/init/revocation_test.go index cfe99ce..c15af01 100644 --- a/internal/init/revocation_test.go +++ b/internal/init/revocation_test.go @@ -83,7 +83,7 @@ func newIssuedFixture(t *testing.T, leafNotAfter time.Time) issuedFixture { leaf, _ := testCert(t, "leaf.example.org", issuing, issuingKey, false, leafNotAfter) chain := [][]byte{issuing.Raw, root.Raw} - if err := issuedRecorder(store)(context.Background(), leaf.Raw, "server"); err != nil { + if err := IssuedRecorder(store)(context.Background(), leaf.Raw, "server"); err != nil { t.Fatalf("record issued: %v", err) } r := &nodeRevoker{store: store, chain: func(context.Context) ([][]byte, error) { return chain, nil }} diff --git a/internal/init/run.go b/internal/init/run.go index bab5126..7cf1497 100644 --- a/internal/init/run.go +++ b/internal/init/run.go @@ -485,7 +485,7 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) { crlDur := time.Duration(nonzero(cfg.PKI.CRLNextUpdateHours, defaultCRLNextUpdateHours)) * time.Hour crlBuilder := revocation.NewCRLBuilder(revStore, crlDur) ocspResp := revocation.NewOCSPResponder(revStore) - caSigner.WithPreflight(preflight.Ensure).WithRecorder(issuedRecorder(revStore)) + caSigner.WithPreflight(preflight.Ensure).WithRecorder(IssuedRecorder(revStore)) revoker := &nodeRevoker{store: revStore, crlBuilder: crlBuilder, load: keyLoader, issuer: issuerFunc, chain: func(ctx context.Context) ([][]byte, error) { id, err := store.Identity(ctx) @@ -801,19 +801,9 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) { // decides the extensions and the certificate is recorded before it is // handed back; the ACME layer only decides which names were proved. if cfg.PKI.ACME != nil { - acmeOpts, aerr := acmeOptions(cfg.PKI.ACME) - if aerr != nil { - return aerr - } - acmeHandler, herr := acme.NewHandler( - acme.NewStore(cli), - acmeIssuer(caSigner, cfg.PKI.ACME.Profile), - acmeRevoker(revoker), - acme.HTTP01Validator(0), - acmeOpts, - ) + acmeHandler, herr := NewACMEHandler(cli, caSigner, acmeRevoker(revoker), cfg.PKI.ACME) if herr != nil { - return fmt.Errorf("init: build the ACME handler: %w", herr) + return herr } acmeAddr := fmt.Sprintf(":%d", nonzero(cfg.PKI.ACME.HTTPPort, defaultACMEHTTPPort)) stopACME, serr := acme.Serve(ctx, acmeAddr, acmeHandler) @@ -826,7 +816,7 @@ func boot(ctx context.Context, shutdown *shutdownRequests) (err error) { _ = stopACME(shutdownCtx) }() log.Printf("ACME listener up: %s (base=%s profile=%s external_account_binding=%t)", - acmeAddr, cfg.PKI.ACME.BaseURL, cfg.PKI.ACME.Profile, acmeOpts.ExternalAccountRequired) + acmeAddr, cfg.PKI.ACME.BaseURL, cfg.PKI.ACME.Profile, !cfg.PKI.ACME.AllowAnonymousAccounts) } // 12d. EST (RFC 7030) enrolment listener. Unlike ACME it terminates TLS diff --git a/test/kind/kind-config.yaml b/test/kind/kind-config.yaml new file mode 100644 index 0000000..5210caf --- /dev/null +++ b/test/kind/kind-config.yaml @@ -0,0 +1,15 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + # Envoy binds hostPort 80/443 on the node; these map them to the host's + # loopback, where the node's http-01 fetch and the test's HTTPS check go. + extraPortMappings: + - containerPort: 80 + hostPort: 80 + listenAddress: 127.0.0.1 + protocol: TCP + - containerPort: 443 + hostPort: 443 + listenAddress: 127.0.0.1 + protocol: TCP diff --git a/test/kind/run.sh b/test/kind/run.sh new file mode 100755 index 0000000..8d00b77 --- /dev/null +++ b/test/kind/run.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +# Stand up a kind cluster with cert-manager, Contour and a small test app, then +# run internal/e2e's TestKindCertManagerACME against it: cert-manager enrols +# over ACME (http-01) with an in-process CryptOS Intermediate running on this +# host, and the test checks the chain, the SANs, the node's issued set and a +# forced renewal. Entry point for `task e2e:kind` and the kind CI workflow. +# +# Linux only: the cluster reaches the node over the kind docker bridge, and the +# node's http-01 fetch reaches the cluster's ingress on 127.0.0.1:80. It skips +# (exit 0) when docker or the pinned kind cannot be had, and fails on anything +# else, including a checksum mismatch. +# +# Needs: docker, curl, sha256sum, go, and sudo for one /etc/hosts line (added +# and removed by this script). KEEP_CLUSTER=1 leaves the cluster up afterwards. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$here/../.." && pwd)" +# shellcheck source=test/kind/versions.env +source "$here/versions.env" + +cluster="${CRYPTOS_E2E_KIND_CLUSTER:-cryptos-e2e}" +hostname="whoami.cryptos.test" +hosts_marker="# cryptos-e2e-kind" +cache="${XDG_CACHE_HOME:-$HOME/.cache}/cryptos-e2e-kind" +work="$(mktemp -d)" +kubeconfig="$work/kubeconfig" + +log() { printf '[e2e:kind] %s\n' "$*" >&2; } +skip() { + log "SKIP: $*" + exit 0 +} + +# ---- preflight ------------------------------------------------------------- + +[ "$(uname -s)" = "Linux" ] || skip "needs a Linux host (the cluster reaches the node over the docker bridge); this is $(uname -s)" +command -v docker >/dev/null 2>&1 || skip "docker is not installed" +docker info >/dev/null 2>&1 || skip "the docker daemon is not reachable (is it running, and can this user use it?)" +for tool in curl sha256sum go; do + command -v "$tool" >/dev/null 2>&1 || { log "missing required tool: $tool"; exit 1; } +done + +case "$(uname -m)" in + x86_64) arch=amd64 kind_sha="$KIND_SHA256_AMD64" kubectl_sha="$KUBECTL_SHA256_AMD64" ;; + aarch64 | arm64) arch=arm64 kind_sha="$KIND_SHA256_ARM64" kubectl_sha="$KUBECTL_SHA256_ARM64" ;; + *) skip "no pinned kind/kubectl for $(uname -m)" ;; +esac + +# fetch downloads url to dest unless dest already matches sha. A failed download +# returns 2 so the caller can decide between skipping and failing; a checksum +# mismatch is always fatal. +fetch() { + local url="$1" dest="$2" sha="$3" + if [ -f "$dest" ] && echo "$sha $dest" | sha256sum -c --status -; then + log "cached: $dest" + return 0 + fi + log "downloading $url" + mkdir -p "$(dirname "$dest")" + if ! curl -fsSL --retry 3 -o "$dest.tmp" "$url"; then + rm -f "$dest.tmp" + return 2 + fi + if ! echo "$sha $dest.tmp" | sha256sum -c --status -; then + log "checksum mismatch for $url (want $sha, got $(sha256sum "$dest.tmp" | cut -d' ' -f1))" + rm -f "$dest.tmp" + exit 1 + fi + mv "$dest.tmp" "$dest" +} + +kind_bin="$cache/kind-$KIND_VERSION-$arch" +fetch "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-$arch" "$kind_bin" "$kind_sha" || + skip "kind $KIND_VERSION is not cached and could not be downloaded" +chmod +x "$kind_bin" + +kubectl_bin="$cache/kubectl-$KUBECTL_VERSION-$arch" +fetch "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/$arch/kubectl" "$kubectl_bin" "$kubectl_sha" || + { log "kubectl $KUBECTL_VERSION could not be downloaded"; exit 1; } +chmod +x "$kubectl_bin" + +cm_yaml="$cache/cert-manager-$CERT_MANAGER_VERSION.yaml" +fetch "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" "$cm_yaml" "$CERT_MANAGER_SHA256" || + { log "cert-manager $CERT_MANAGER_VERSION manifest could not be downloaded"; exit 1; } + +contour_yaml="$cache/contour-$CONTOUR_VERSION.yaml" +fetch "https://raw.githubusercontent.com/projectcontour/contour/$CONTOUR_VERSION/examples/render/contour.yaml" "$contour_yaml" "$CONTOUR_SHA256" || + { log "Contour $CONTOUR_VERSION manifest could not be downloaded"; exit 1; } + +kind() { "$kind_bin" "$@"; } +kubectl() { "$kubectl_bin" --kubeconfig "$kubeconfig" "$@"; } + +# ---- teardown -------------------------------------------------------------- + +added_hosts=0 +cleanup() { + local rc=$? + if [ "$added_hosts" = 1 ]; then + log "removing the $hostname line from /etc/hosts" + sudo sed -i "/$hosts_marker\$/d" /etc/hosts || true + fi + if [ "${KEEP_CLUSTER:-0}" = 1 ]; then + log "KEEP_CLUSTER=1: leaving cluster $cluster up (kubeconfig: $kubeconfig)" + else + kind delete cluster --name "$cluster" >/dev/null 2>&1 || true + rm -rf "$work" + fi + exit "$rc" +} +trap cleanup EXIT + +# ---- name resolution on the host -------------------------------------------- + +# The node's http-01 validator resolves the name the ordinary way, so the host +# must resolve it to the ingress on loopback. No validator bypass exists, and +# none is added for this test. +if ! grep -qE "^[^#]*[[:space:]]$hostname([[:space:]]|\$)" /etc/hosts; then + if ! sudo -n true 2>/dev/null; then + log "add this line to /etc/hosts and rerun (sudo is needed to add it for you):" + log " 127.0.0.1 $hostname" + exit 1 + fi + echo "127.0.0.1 $hostname $hosts_marker" | sudo tee -a /etc/hosts >/dev/null + added_hosts=1 + log "added 127.0.0.1 $hostname to /etc/hosts" +fi + +# ---- cluster --------------------------------------------------------------- + +kind delete cluster --name "$cluster" >/dev/null 2>&1 || true +log "creating kind cluster $cluster ($KIND_NODE_IMAGE)" +kind create cluster --name "$cluster" --image "$KIND_NODE_IMAGE" \ + --config "$here/kind-config.yaml" --kubeconfig "$kubeconfig" --wait 120s + +host_ip="$(docker inspect -f '{{with index .NetworkSettings.Networks "kind"}}{{.Gateway}}{{end}}' "$cluster-control-plane")" +[ -n "$host_ip" ] || { log "could not read the kind network gateway"; exit 1; } +log "pods reach this host at $host_ip" + +log "installing Contour $CONTOUR_VERSION" +kubectl apply -f "$contour_yaml" >/dev/null +log "installing cert-manager $CERT_MANAGER_VERSION" +kubectl apply -f "$cm_yaml" >/dev/null +log "deploying the whoami test app" +sed -e "s|WHOAMI_IMAGE|$WHOAMI_IMAGE|" -e "s|E2E_HOSTNAME|$hostname|g" "$here/whoami.yaml" | kubectl apply -f - >/dev/null + +kubectl -n projectcontour wait --for=condition=complete job --all --timeout=180s +kubectl -n projectcontour rollout status deployment/contour --timeout=180s +kubectl -n projectcontour rollout status daemonset/envoy --timeout=180s +kubectl -n cert-manager wait --for=condition=Available deployment --all --timeout=180s +kubectl -n cryptos-e2e rollout status deployment/whoami --timeout=180s + +# cert-manager checks the challenge URL from inside the cluster before it asks +# the CA to validate, so the name must resolve there too: to Envoy's service. +envoy_ip="$(kubectl -n projectcontour get service envoy -o jsonpath='{.spec.clusterIP}')" +log "CoreDNS: $hostname -> $envoy_ip (Envoy)" +corefile="$(kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}')" +corefile="$(printf '%s\n' "$corefile" | awk -v ip="$envoy_ip" -v h="$hostname" ' + { print } + /^\.:53 \{/ { print " hosts {\n " ip " " h "\n fallthrough\n }" }')" +kubectl -n kube-system create configmap coredns --from-literal=Corefile="$corefile" --dry-run=client -o yaml | + kubectl apply -f - >/dev/null +kubectl -n kube-system rollout restart deployment/coredns >/dev/null +kubectl -n kube-system rollout status deployment/coredns --timeout=120s + +# ---- the test -------------------------------------------------------------- + +log "running TestKindCertManagerACME" +cd "$root" +CRYPTOS_E2E_KIND=1 \ + CRYPTOS_E2E_KIND_HOST_IP="$host_ip" \ + CRYPTOS_E2E_KIND_HOSTNAME="$hostname" \ + CRYPTOS_E2E_KIND_KUBECONFIG="$kubeconfig" \ + CRYPTOS_E2E_KIND_KUBECTL="$kubectl_bin" \ + go test -count=1 -v -timeout 15m -run '^TestKindCertManagerACME$' ./internal/e2e/ diff --git a/test/kind/versions.env b/test/kind/versions.env new file mode 100644 index 0000000..8fabc68 --- /dev/null +++ b/test/kind/versions.env @@ -0,0 +1,27 @@ +# Pinned tools, images and manifests for the kind ACME end-to-end test +# (test/kind/run.sh, `task e2e:kind`). Every download is checked against the +# SHA-256 here and every image is pinned by digest; bump deliberately and +# re-verify each value against the upstream release. + +# kind: https://github.com/kubernetes-sigs/kind/releases/tag/v0.33.0 +KIND_VERSION=v0.33.0 +KIND_SHA256_AMD64=aee6151561422756b764a4ae28e7f44cda5af5a9eead3cc9985112b1de8d8e0d +KIND_SHA256_ARM64=20022bee6cfcd5086cb7234d218e3454e6090022f2a8f55d1fa7fcf42c3867a2 +# The node image kind v0.33.0 lists for Kubernetes v1.37.0. +KIND_NODE_IMAGE=kindest/node:v1.37.0@sha256:a1ed56cfb0e7b93589bdf97c8cd566405a265939e3620fc4f5de89adff580ae5 + +# kubectl matching the node's Kubernetes version (dl.k8s.io publishes .sha256). +KUBECTL_VERSION=v1.37.0 +KUBECTL_SHA256_AMD64=6129359f4e1f3848a5572ccb0b26cf28b8ca08cef38c95a765b2f64a2c961a2f +KUBECTL_SHA256_ARM64=922df28df248cc00a9e025f947704f1d1482de64ece54cfe57e61f19eaf1eef3 + +# cert-manager static manifest from the v1.21.2 release assets. +CERT_MANAGER_VERSION=v1.21.2 +CERT_MANAGER_SHA256=e03b668ec8675214af6b0a671699d088f2601fa3878e0dbe1b41d3feafd1879f + +# Contour's rendered quickstart manifest (Envoy DaemonSet on hostPort 80/443). +CONTOUR_VERSION=v1.33.7 +CONTOUR_SHA256=96caefecf8272bde6467528514a3ead14bf165176ad6711bc11ab141c1ae62c8 + +# The test app behind the ingress. +WHOAMI_IMAGE=docker.io/traefik/whoami:v1.12.0@sha256:c4717a8d1f0134a7444e24f881160e033991f23027c6c5a9a3f8fd22e70d1d44 diff --git a/test/kind/whoami.yaml b/test/kind/whoami.yaml new file mode 100644 index 0000000..9c3e16d --- /dev/null +++ b/test/kind/whoami.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cryptos-e2e +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: whoami + namespace: cryptos-e2e +spec: + replicas: 1 + selector: + matchLabels: + app: whoami + template: + metadata: + labels: + app: whoami + spec: + containers: + - name: whoami + image: WHOAMI_IMAGE + args: ["--port", "8080"] + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: whoami + namespace: cryptos-e2e +spec: + selector: + app: whoami + ports: + - port: 80 + targetPort: 8080 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: whoami + namespace: cryptos-e2e +spec: + ingressClassName: contour + tls: + - hosts: [E2E_HOSTNAME] + secretName: whoami-tls + rules: + - host: E2E_HOSTNAME + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: whoami + port: + number: 80