diff --git a/.github/workflows/ci-e2e-image.yml b/.github/workflows/ci-e2e-image.yml new file mode 100644 index 0000000..85741c6 --- /dev/null +++ b/.github/workflows/ci-e2e-image.yml @@ -0,0 +1,199 @@ +name: Full-image suite (QEMU Root and Intermediate) + +# Boots the real CryptOS image in QEMU (swtpm, OVMF) as a Root and an +# Intermediate on a GitHub-hosted runner and runs the suite of named steps in +# test/integration/image_suite_test.go against them over the mTLS API: +# ceremony, sign-subordinate, leaves, nginx with OCSP stapling, revocation, +# the protocol switch, ACME with cert-manager in kind, EST, re-certify, escrow +# and the console reset. Each step's result and the coverage the run reached +# go into the job summary. See test/image/run.sh. +# +# Slow, so not on every pull request: nightly, on pull requests that touch +# the image build, the boot or the protocol and config code, and on manual +# dispatch. It is not a required check; the fast in-process kind test +# (ci-kind-acme.yml) stays the pull-request check for ACME. +# +# The image job builds a coverage-instrumented variant of the image +# (test/image/build.sh) and caches it by the hash of everything that goes into +# it, so a run whose inputs did not change reuses the last build instead of +# rebuilding. The kernel build shares ci-image.yml's cache. No release or CI +# image is ever built with -cover. +on: + schedule: + - cron: "37 6 * * *" + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + paths: + - "build/**" + - "cmd/init/**" + - "cmd/cryptos-console/**" + - "cmd/cryptos-switchroot/**" + - "internal/init/**" + - "internal/acme/**" + - "internal/est/**" + - "internal/config/**" + - "internal/revocation/**" + - "test/image/**" + - "test/integration/**" + - ".github/workflows/ci-e2e-image.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + image: + name: Build the coverage-instrumented test image + if: github.event.pull_request.draft != true + runs-on: ubuntu-latest + timeout-minutes: 40 + outputs: + key: ${{ steps.key.outputs.key }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # buildinfo.sh describes the build from the tags. + fetch-depth: 0 + + - name: Image cache key + id: key + run: echo "key=e2e-image-uki-${{ hashFiles('build/**', 'cmd/**', 'internal/**', 'go.mod', 'go.sum', 'test/image/build.sh') }}" >>"$GITHUB_OUTPUT" + + - name: Reuse a cached test image + id: cached + uses: actions/cache/restore@v4 + with: + path: build/out/cryptos-amd64-e2e* + key: ${{ steps.key.outputs.key }} + + - name: Set up Go + if: steps.cached.outputs.cache-hit != 'true' + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Install the image toolchain + if: steps.cached.outputs.cache-hit != 'true' + run: | + set -euo pipefail + sudo apt-get update + # The same list as ci-image.yml (see the notes there). + sudo apt-get install -y --no-install-recommends \ + build-essential bc flex bison libelf-dev libssl-dev xz-utils \ + squashfs-tools systemd-ukify systemd-boot-efi cpio python3-pefile + + - name: Cache the kernel build + if: steps.cached.outputs.cache-hit != 'true' + uses: actions/cache@v4 + with: + path: build/.work/kernel + key: kernel-amd64-${{ hashFiles('build/ci/versions.env', 'build/kernel/cryptos.config') }} + + - name: Cache the static tools + if: steps.cached.outputs.cache-hit != 'true' + uses: actions/cache@v4 + with: + path: | + build/out/cryptsetup-amd64 + build/out/mke2fs-amd64 + build/out/sgdisk-amd64 + build/out/mkfs.vfat-amd64 + key: e2e-static-tools-amd64-${{ hashFiles('build/ci/versions.env', 'build/cryptsetup/**', 'build/e2fsprogs/**', 'build/gptfdisk/**', 'build/dosfstools/**') }} + + - name: Build the test image + if: steps.cached.outputs.cache-hit != 'true' + run: bash test/image/build.sh + + - name: Save the test image + if: steps.cached.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: build/out/cryptos-amd64-e2e* + key: ${{ steps.key.outputs.key }} + + - name: Hand the image to the suite + uses: actions/upload-artifact@v4 + with: + name: e2e-image-uki + path: build/out/cryptos-amd64-e2e* + retention-days: 3 + if-no-files-found: error + + suite: + name: Boot the image and run the suite + needs: image + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Fetch the test image + uses: actions/download-artifact@v4 + with: + name: e2e-image-uki + path: build/out + + # Hosted Linux runners expose /dev/kvm, but only to root until a udev + # rule opens it up (the GitHub-documented way to use it). + - name: Enable KVM + run: | + set -euo pipefail + if [ -e /dev/kvm ]; then + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | + sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + ls -l /dev/kvm + echo "KVM: available" >>"$GITHUB_STEP_SUMMARY" + else + echo "KVM: not available on this runner; the VMs run under TCG" >>"$GITHUB_STEP_SUMMARY" + fi + + - name: Install QEMU, swtpm, OVMF and mtools + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + qemu-system-x86 qemu-utils ovmf swtpm swtpm-tools mtools gdisk + qemu-system-x86_64 --version | head -1 + swtpm --version + + - name: Cache kind, kubectl and manifests + uses: actions/cache@v4 + with: + path: ~/.cache/cryptos-e2e-kind + key: kind-acme-${{ runner.arch }}-${{ hashFiles('test/kind/versions.env') }} + + - name: Run the suite + env: + E2E_OUT: ${{ runner.temp }}/e2e-image + run: bash test/image/run.sh + + - name: Keep the logs, serial consoles and coverage report + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-image-results + path: | + ${{ runner.temp }}/e2e-image/*.md + ${{ runner.temp }}/e2e-image/*.log + ${{ runner.temp }}/e2e-image/coverage* + ${{ runner.temp }}/e2e-image/vm-*/serial.log + retention-days: 14 + if-no-files-found: warn diff --git a/README.md b/README.md index c7badd7..9b741b7 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ internal/ bootstrap/ # bootstrap admin cert loading + first-ceremony rotation build/ # kernel config, UKI assembly recipes, SquashFS templates test/kind/ # kind + cert-manager ACME end-to-end harness (task e2e:kind) +test/image/ # full-image suite: build, run and coverage scripts (task e2e:image) +test/integration/ # QEMU + swtpm tests against the booted image, including the full-image suite testdata/configs/ # sample machine configs ``` @@ -57,10 +59,13 @@ task ci # fmt + lint + vet + test + build (both binaries) task build # produces bin/init and bin/cryptosctl, stamped with the build identity task license # re-inject Apache 2.0 headers via golic task e2e:kind # Linux + docker: cert-manager in kind gets a certificate over ACME +task e2e:image # Linux + KVM + docker: boot the image as a Root and an Intermediate and run the full suite ``` `task e2e:kind` builds a Root and an ACME-serving Intermediate in-process (software keys, no TPM), stands up a kind cluster with cert-manager and Contour, and checks that a `Certificate` goes Ready with a chain to the root and renews to a new serial. It downloads pinned, checksum-checked kind, kubectl and manifests ([`test/kind/versions.env`](test/kind/versions.env)), needs sudo once to add a `/etc/hosts` line for the test name, and skips when docker isn't available. +`task e2e:image` builds a coverage-instrumented variant of the image ([`test/image/build.sh`](test/image/build.sh): the same kernel and rootfs, with `/init` built with `-cover` and a throwaway upgrade anchor, plus a signed successor image for the upgrade step; release and CI images are never built with `-cover`), boots it in QEMU with swtpm and OVMF as a software-key Root and a TPM-key Intermediate, and runs the steps in [`test/integration/image_suite_test.go`](test/integration/image_suite_test.go) against them with `cryptosctl`: the ceremony, sign-subordinate, leaf issuance, nginx with OCSP stapling, revocation, the ACME and EST protocol switch, cert-manager over ACME in kind, EST enrolment, re-certify, escrow, an in-place image upgrade on both nodes and the console reset. Each step reports pass, fail or skip in `summary.md`, and the merged coverage lands in `coverage.html`. It needs qemu, swtpm, OVMF, mtools, sgdisk, docker and sudo for one `/etc/hosts` line, and skips on a host without them. + `bin/cryptosctl version` prints the version, commit, and build date the binary was built from (`git describe --tags --always --dirty`; see [`build/README.md`](build/README.md#build-identity)); pass `--endpoint` to also show the node's version. The image pipeline (`task image` — hardened kernel build, SquashFS rootfs, UKI assembly + Secure Boot signing) has draft recipes under `build/` that run on a Linux build host; see [`build/README.md`](build/README.md). They are written but not yet executed end to end. The QEMU + `swtpm` integration harness lands in a subsequent PR. @@ -102,7 +107,7 @@ GitHub Actions: - **`ci-kind-acme`** ([`ci-kind-acme.yml`](.github/workflows/ci-kind-acme.yml)) — `test/kind/run.sh` (the `task e2e:kind` harness) on pull requests that touch the ACME, config, node or e2e code, on a GitHub-hosted runner. Drafts are skipped, and it isn't a required check. -The QEMU + `swtpm` integration boot is run on a real host by the operator, not in CI. +- **`ci-e2e-image`** ([`ci-e2e-image.yml`](.github/workflows/ci-e2e-image.yml)) — the full-image suite (`task e2e:image`) on a GitHub-hosted runner, with KVM when the runner has it: nightly, on pull requests that touch the image build, the boot, or the protocol and config code, and on manual dispatch. The step table and per-package coverage go into the job summary and the HTML report into the run's artifacts. Drafts are skipped, and it isn't a required check. ## 🔑 Management surfaces diff --git a/Taskfile.yml b/Taskfile.yml index 88e8d34..c931d18 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -184,6 +184,12 @@ tasks: cmds: - bash test/kind/run.sh + e2e:image: + desc: "[Linux] Full-image suite: boot the image in QEMU as a Root and an Intermediate and drive every step with cryptosctl (needs KVM, qemu, swtpm, OVMF, docker; skips without them)" + cmds: + - bash test/image/build.sh + - bash test/image/run.sh + test:integration: desc: "[Linux] Phase 1 QEMU + swtpm end-to-end acceptance (needs the toolchain; see test/integration)" cmds: diff --git a/cmd/init/coverflush_e2ecover_linux.go b/cmd/init/coverflush_e2ecover_linux.go new file mode 100644 index 0000000..7604699 --- /dev/null +++ b/cmd/init/coverflush_e2ecover_linux.go @@ -0,0 +1,163 @@ +//go:build e2ecover + +package main + +/* +Copyright The CryptOS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file is compiled only into the coverage-instrumented test image that +// the full-image suite boots (test/image/build.sh sets -tags=e2ecover together +// with -cover). No release or CI image build sets the tag. +// +// PID 1 never exits, so Go never writes its coverage counters on its own. The +// suite attaches a small FAT disk with the virtio serial coverSerial; this +// mounts it and rewrites the counters there every few seconds, so whatever +// the node ran up to its last flush survives a reboot, a power-off or the +// harness killing the VM. The harness reads the disk after QEMU exits. + +import ( + "io" + "log" + "os" + "path/filepath" + "runtime/coverage" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +const ( + coverSerial = "cryptos-cover" + coverMount = "/run/cryptos-cover" + coverStaging = "/run/cryptos-cover-staging" + coverInterval = 2 * time.Second +) + +func init() { + go flushCoverage() +} + +func flushCoverage() { + var dir string + for dir == "" { + time.Sleep(time.Second) + dir = mountCoverDisk() + } + log.Printf("coverage: writing counters to %s every %s", dir, coverInterval) + if err := coverage.WriteMetaDir(dir); err != nil { + log.Printf("coverage: write meta-data: %v (is this binary built with -cover?)", err) + return + } + // Logged once: init logs to /dev/kmsg, which the kernel rate-limits, so a + // repeating line would crowd out the node's own boot messages. + var last string + reported := false + for { + name, err := writeCounters(dir) + if err != nil { + if !reported { + log.Printf("coverage: write counters: %v", err) + reported = true + } + } else { + if last != "" && last != name { + _ = os.Remove(filepath.Join(dir, last)) + } + last = name + } + time.Sleep(coverInterval) + } +} + +// mountCoverDisk finds the virtio disk carrying coverSerial and mounts it +// synchronously on coverMount. It returns "" until /sys, /dev and /run are up +// and the disk is there. +func mountCoverDisk() string { + entries, err := os.ReadDir("/sys/block") + if err != nil { + return "" + } + for _, e := range entries { + serial, err := os.ReadFile(filepath.Join("/sys/block", e.Name(), "serial")) + if err != nil || strings.TrimSpace(string(serial)) != coverSerial { + continue + } + if err := os.MkdirAll(coverMount, 0o700); err != nil { + return "" + } + dev := filepath.Join("/dev", e.Name()) + if err := unix.Mount(dev, coverMount, "vfat", unix.MS_SYNCHRONOUS|unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV, ""); err != nil { + log.Printf("coverage: mount %s on %s: %v", dev, coverMount, err) + return "" + } + return coverMount + } + return "" +} + +// writeCounters snapshots the counters into a staging directory on the /run +// tmpfs and copies the new file onto the disk under a temporary name before +// renaming it, so a VM killed mid-copy leaves the previous snapshot whole. +func writeCounters(dir string) (string, error) { + if err := os.RemoveAll(coverStaging); err != nil { + return "", err + } + if err := os.MkdirAll(coverStaging, 0o700); err != nil { + return "", err + } + if err := coverage.WriteCountersDir(coverStaging); err != nil { + return "", err + } + files, err := os.ReadDir(coverStaging) + if err != nil { + return "", err + } + var name string + for _, f := range files { + if strings.HasPrefix(f.Name(), "covcounters.") { + name = f.Name() + } + } + if name == "" { + return "", os.ErrNotExist + } + src, err := os.Open(filepath.Join(coverStaging, name)) + if err != nil { + return "", err + } + defer func() { _ = src.Close() }() + tmp := filepath.Join(dir, "partial") + dst, err := os.Create(tmp) + if err != nil { + return "", err + } + if _, err := io.Copy(dst, src); err != nil { + _ = dst.Close() + return "", err + } + if err := dst.Sync(); err != nil { + _ = dst.Close() + return "", err + } + if err := dst.Close(); err != nil { + return "", err + } + if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil { + return "", err + } + return name, nil +} diff --git a/docs/image-upgrade.md b/docs/image-upgrade.md index 3897300..49ea585 100644 --- a/docs/image-upgrade.md +++ b/docs/image-upgrade.md @@ -46,11 +46,13 @@ reinstall. > [!WARNING] > **This has not been exercised on production hardware yet.** The verification, -> slot management, RPC and CLI paths are covered by tests; the actual -> `mount`/reboot sequence on a real ESP is not something a test can cover. -> The same goes for the TPM reseal: it is tested against a TPM simulator and -> a model of the stub's measurement, not yet across a real reboot on a vTPM. -> Do the first run on a node you can reach physically. +> slot management, RPC and CLI paths are covered by tests, and the full-image +> suite (`ci-e2e-image.yml`) stages and activates a successor image on a +> TPM-backed node and on a `nodeid` node booted in QEMU with swtpm and Secure +> Boot off: the TPM node reseals, and both reboot onto the new image and +> unlock their state partition. That is a software TPM on a virtual ESP, not +> a vTPM or physical TPM with Secure Boot on. Do the first run on a node you +> can reach physically. > [!NOTE] > `cryptosctl` runs on Linux and macOS today. A Windows build is coming. diff --git a/internal/revocation/http.go b/internal/revocation/http.go index 97d7dc1..91aac92 100644 --- a/internal/revocation/http.go +++ b/internal/revocation/http.go @@ -24,6 +24,7 @@ import ( "io" "net" "net/http" + "net/url" "strings" "time" @@ -62,12 +63,14 @@ func NewHandler(crl func(context.Context) ([]byte, error), ocspFn func(context.C } // Routes returns the anonymous HTTP mux: GET /crl yields the DER CRL as -// application/pkix-crl; /ocsp yields an application/ocsp-response; GET +// application/pkix-crl; POST /ocsp and GET /ocsp/ +// (RFC 6960 appendix A.1) yield an application/ocsp-response; GET // CACertPath yields the node's DER CA certificate as application/pkix-cert. func (h *Handler) Routes() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/crl", h.handleCRL) mux.HandleFunc("/ocsp", h.handleOCSP) + mux.HandleFunc("/ocsp/", h.handleOCSP) mux.HandleFunc(CACertPath, h.handleCACert) return mux } @@ -144,7 +147,12 @@ func readOCSPRequest(r *http.Request) ([]byte, error) { case http.MethodPost: return io.ReadAll(io.LimitReader(r.Body, maxOCSPRequestBytes)) case http.MethodGet: - encoded := strings.TrimPrefix(r.URL.Path, "/ocsp/") + // The escaped path, because the base64 may hold "/" that the client + // percent-encoded and a decoded path would split. + encoded, err := url.PathUnescape(strings.TrimPrefix(r.URL.EscapedPath(), "/ocsp/")) + if err != nil { + return nil, err + } return base64.StdEncoding.DecodeString(encoded) default: return nil, errMethodNotAllowed diff --git a/internal/revocation/http_test.go b/internal/revocation/http_test.go index 2dbeaeb..7cbbffd 100644 --- a/internal/revocation/http_test.go +++ b/internal/revocation/http_test.go @@ -19,11 +19,13 @@ limitations under the License. import ( "bytes" "context" + "encoding/base64" "errors" "io" "net" "net/http" "net/http/httptest" + "net/url" "testing" ) @@ -142,6 +144,48 @@ func TestOCSPEndpointPOSTServesResponse(t *testing.T) { } } +// RFC 6960 appendix A.1: a GET request is the URL-encoded base64 of the DER +// request appended to the responder URL. nginx staples this way. The request +// bytes are chosen so the base64 holds "/", "+" and "=", which the client +// percent-encodes and the responder must decode. +func TestOCSPEndpointGETServesResponse(t *testing.T) { + want := []byte{0x30, 0x03} + reqDER := []byte{0xff, 0xff, 0xfb, 0xef} + var gotReq []byte + h := NewHandler( + func(context.Context) ([]byte, error) { return nil, nil }, + func(_ context.Context, der []byte) ([]byte, error) { + gotReq = der + return want, nil + }, + nil, + ) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + encoded := base64.StdEncoding.EncodeToString(reqDER) + for _, path := range []string{ + "/ocsp/" + url.QueryEscape(encoded), + "/ocsp/" + url.PathEscape(encoded), + } { + gotReq = nil + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != "application/ocsp-response" { + t.Fatalf("GET %s: status=%d ctype=%s", path, resp.StatusCode, resp.Header.Get("Content-Type")) + } + if !bytes.Equal(gotReq, reqDER) { + t.Fatalf("GET %s: handler received %x, want %x", path, gotReq, reqDER) + } + if !bytes.Equal(body, want) { + t.Fatalf("GET %s: body=%x, want %x", path, body, want) + } + } +} + func TestCACertEndpointServesDER(t *testing.T) { want := []byte{0x30, 0x02} // stand-in certificate DER noCRL := func(context.Context) ([]byte, error) { return nil, nil } diff --git a/test/image/build.sh b/test/image/build.sh new file mode 100755 index 0000000..c4be594 --- /dev/null +++ b/test/image/build.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Build the image the full-image suite boots (test/image/run.sh): the same +# kernel, static tools and rootfs as `task image:debug`, unsigned and with no +# upgrade anchor, except that /init is built with Go coverage instrumentation +# (-cover and the e2ecover tag, cmd/init/coverflush_e2ecover_linux.go). The +# kernel command line is the qemu-dev profile, so the boot log and the console +# are on the serial line the suite reads and types into. +# +# For the upgrade step it also stamps a per-build upgrade anchor (a throwaway +# RSA key made here and discarded, like ci-image.yml's smoke build) and builds +# a second image with a different version string, both with detached +# signatures, so a node booted from the first can be upgraded to the second. +# +# This build is for the suite only. Release and CI images never set -cover or +# the e2ecover tag. Output in build/out: cryptos-amd64-e2e.uki and +# cryptos-amd64-e2e-next.uki, each with a .sig. +# +# Linux only. Needs the image toolchain (build/README.md) and docker for the +# static tools, which are reused when build/out already holds them. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$here/../.." && pwd)" +arch=amd64 +out="$root/build/out" +uki="$out/cryptos-$arch-e2e.uki" + +log() { printf '[e2e:image build] %s\n' "$*" >&2; } + +[ "$(uname -s)" = "Linux" ] || { log "SKIP: the image builds on a Linux host; this is $(uname -s)"; exit 0; } + +cd "$root" +started=$SECONDS + +log "kernel (reuses build/.work/kernel when it is there)" +PLATFORM='' bash build/kernel/build.sh "$arch" + +for tool in cryptsetup:cryptsetup e2fsprogs:mke2fs gptfdisk:sgdisk dosfstools:mkfs.vfat; do + dir="${tool%%:*}" bin="${tool##*:}" + if [ -x "$out/$bin-$arch" ]; then + log "reusing $out/$bin-$arch" + else + log "building $bin" + bash "build/$dir/build.sh" "$arch" + fi +done + +anchor="$root/build/.work/e2e-anchor" +rm -rf "$anchor" +mkdir -p "$anchor" +trap 'rm -rf "$anchor"' EXIT +log "throwaway upgrade anchor" +go run ./cmd/cryptos-sbkey --out-dir "$anchor" --cn "CryptOS full-image suite (throwaway)" --days 2 >/dev/null + +version="$(git describe --tags --always --dirty 2>/dev/null || echo dev)" +build_one() { + local name="$1" ver="$2" + log "rootfs $ver with a coverage-instrumented init" + CRYPTOS_VERSION="$ver" SB_CERT="$anchor/sb.crt" STATEKEY=tpm \ + GOFLAGS="-cover -covermode=atomic -coverpkg=github.com/CryptOS-PKI/cryptos/... -tags=e2ecover" \ + bash build/squashfs/build.sh "$arch" + log "UKI $name (qemu-dev command line)" + CRYPTOS_VERSION="$ver" bash build/uki/assemble.sh "$arch" qemu-dev + cp "$out/cryptos-$arch.uki.unsigned" "$out/$name" + openssl dgst -sha256 -sign "$anchor/sb.key" -out "$out/$name.sig" "$out/$name" +} +build_one "cryptos-$arch-e2e.uki" "$version" +build_one "cryptos-$arch-e2e-next.uki" "$version-next" +log "wrote $uki and its successor in $((SECONDS - started))s" diff --git a/test/image/coverage.sh b/test/image/coverage.sh new file mode 100755 index 0000000..e8cb961 --- /dev/null +++ b/test/image/coverage.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Merge the coverage counters the full-image suite collected and report them: +# one directory per VM (copied off its coverage disk by the suite) plus one for +# the coverage-instrumented cryptosctl. Writes, into : +# coverage.txt the merged profile (go tool cover format) +# coverage.html the HTML report +# coverage-percent.txt go tool covdata percent, one line per package +# coverage-summary.md the per-package table for the job summary +# +# Usage: test/image/coverage.sh +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$here/../.." && pwd)" +counters="${1:?counter root}" +out="${2:?output dir}" + +log() { printf '[e2e:image coverage] %s\n' "$*" >&2; } + +cd "$root" + +# A node can stop mid-write (a reboot or the harness stopping the VM), which +# leaves a short counter file that covdata refuses outright. Check each file +# on its own and set aside the ones that do not read. +check="$(mktemp -d)" +trap 'rm -rf "$check"' EXIT +for f in "$counters"/*/covcounters.*; do + [ -e "$f" ] || continue + rm -rf "${check:?}"/* + cp "$(dirname "$f")"/covmeta.* "$check"/ 2>/dev/null || true + cp "$f" "$check"/ + if ! go tool covdata percent -i="$check" >/dev/null 2>&1; then + log "unreadable counter file set aside: $f" + mv "$f" "$f.unreadable" + fi +done + +dirs=() +for d in "$counters"/*/; do + d="${d%/}" + if compgen -G "$d/covmeta.*" >/dev/null; then + dirs+=("$d") + log "$(basename "$d"): $(find "$d" -name 'covcounters.*' | wc -l) counter file(s)" + else + log "$(basename "$d"): no coverage meta-data, left out" + fi +done +if [ "${#dirs[@]}" -eq 0 ]; then + log "no coverage data found under $counters" + printf '## Coverage\n\nNo coverage data came back from the run.\n' >"$out/coverage-summary.md" + exit 1 +fi +inputs="$(IFS=,; echo "${dirs[*]}")" +go tool covdata percent -i="$inputs" | sort >"$out/coverage-percent.txt" +go tool covdata textfmt -i="$inputs" -o "$out/coverage.txt" +go tool cover -html="$out/coverage.txt" -o "$out/coverage.html" +total="$(go tool cover -func="$out/coverage.txt" | awk '/^total:/ {print $NF}')" + +{ + printf '## Coverage\n\n' + printf 'Statements covered by the booted nodes'"'"' init and the host'"'"'s cryptosctl: **%s** of the packages those binaries link. ' "$total" + printf 'Sources: %s. The HTML report is in the run'"'"'s artifacts.\n\n' "$(for d in "${dirs[@]}"; do basename "$d"; done | paste -sd, - | sed 's/,/, /g')" + printf '| Package | Covered |\n|---|---|\n' + # covdata prints " coverage: 12.3% of statements". + sed -E 's#^[[:space:]]*github.com/CryptOS-PKI/cryptos/##; s#[[:space:]]+coverage: ([0-9.]+)% of statements#|\1#' "$out/coverage-percent.txt" | + sort -t'|' -k2 -g -r | + awk -F'|' '{ printf "| `%s` | %s%% |\n", $1, $2 }' + printf '\n' +} >"$out/coverage-summary.md" +log "total $total; report in $out/coverage.html" diff --git a/test/image/run.sh b/test/image/run.sh new file mode 100755 index 0000000..44910b4 --- /dev/null +++ b/test/image/run.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# Run the full-image suite: boot the real CryptOS image in QEMU (swtpm, OVMF) +# as a Root and an Intermediate and drive them over the mTLS API with +# cryptosctl, the way an operator would. The steps live in +# test/integration/image_suite_test.go (TestImageSuite); each one reports +# pass, fail or skip in a summary table. Entry point for `task e2e:image` and +# the ci-e2e-image workflow. +# +# This script stands up what the steps need around the VMs: a kind cluster +# with cert-manager and Contour for the ACME step (the pins in +# test/kind/versions.env), a coverage-instrumented cryptosctl, and one +# /etc/hosts line so the node's http-01 fetch, which goes through QEMU's DNS +# proxy to this host's resolver, lands on the cluster ingress. It then merges +# the coverage counters the VMs and cryptosctl wrote. +# +# Linux only. It skips (exit 0) when the host is not Linux or lacks docker, +# qemu, swtpm or OVMF, and fails on anything else. Needs: docker, curl, +# sha256sum, go, openssl, qemu-system-x86_64, swtpm, sgdisk, mtools, OVMF, +# and sudo for the /etc/hosts line (added and removed here). +# +# Environment: +# E2E_IMAGE_UKI the UKI to boot (default build/out/cryptos-amd64-e2e.uki; +# test/image/build.sh makes it) +# E2E_OUT where logs, the step summary and coverage go (default a +# temp dir, printed at the end) +# KEEP_CLUSTER=1 leave the kind cluster up afterwards +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$here/../.." && pwd)" +# shellcheck source=test/kind/versions.env +source "$root/test/kind/versions.env" +# shellcheck source=test/image/versions.env +source "$here/versions.env" + +cluster="${CRYPTOS_E2E_IMAGE_CLUSTER:-cryptos-e2e-image}" +hostname="whoami.cryptos.test" +hosts_marker="# cryptos-e2e-image" +# QEMU user networking puts the host at this address inside each VM +# (net=10.0.0.0/24,host=10.0.0.1); a guest connection to it lands on the +# host's loopback, where kind maps the ingress's ports 80 and 443. +guest_host_ip="10.0.0.1" +cache="${XDG_CACHE_HOME:-$HOME/.cache}/cryptos-e2e-kind" +uki="${E2E_IMAGE_UKI:-$root/build/out/cryptos-amd64-e2e.uki}" + +log() { printf '[e2e:image] %s\n' "$*" >&2; } +skip() { + log "SKIP: $*" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '## Full-image suite\n\nSkipped: %s\n' "$*" >>"$GITHUB_STEP_SUMMARY" + fi + exit 0 +} + +# ---- preflight ------------------------------------------------------------- + +[ "$(uname -s)" = "Linux" ] || skip "needs a Linux host (QEMU with KVM, docker and kind); this is $(uname -s)" +[ "$(uname -m)" = "x86_64" ] || skip "the suite boots the amd64 image; this host is $(uname -m)" +command -v docker >/dev/null 2>&1 || skip "docker is not installed" +docker info >/dev/null 2>&1 || skip "the docker daemon is not reachable (is it running, and can this user use it?)" +for tool in qemu-system-x86_64 swtpm sgdisk mformat mcopy; do + command -v "$tool" >/dev/null 2>&1 || skip "$tool is not installed" +done +ovmf_code="${OVMF_CODE:-}" +ovmf_vars="${OVMF_VARS:-}" +for pair in /usr/share/OVMF/OVMF_CODE_4M.fd:/usr/share/OVMF/OVMF_VARS_4M.fd /usr/share/OVMF/OVMF_CODE.fd:/usr/share/OVMF/OVMF_VARS.fd /usr/share/edk2/ovmf/OVMF_CODE.fd:/usr/share/edk2/ovmf/OVMF_VARS.fd; do + [ -n "$ovmf_code" ] && break + if [ -f "${pair%%:*}" ] && [ -f "${pair##*:}" ]; then + ovmf_code="${pair%%:*}" ovmf_vars="${pair##*:}" + fi +done +[ -n "$ovmf_code" ] || skip "OVMF firmware not found (set OVMF_CODE and OVMF_VARS)" +for tool in curl sha256sum go openssl; do + command -v "$tool" >/dev/null 2>&1 || { log "missing required tool: $tool"; exit 1; } +done +[ -f "$uki" ] || { log "missing $uki (run test/image/build.sh first)"; exit 1; } + +out="${E2E_OUT:-$(mktemp -d)}" +mkdir -p "$out" +out="$(cd "$out" && pwd)" +work="$(mktemp -d)" +kubeconfig="$work/kubeconfig" + +accel=tcg +if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + accel=kvm +else + log "WARNING: /dev/kvm is not usable by this user; the VMs run under TCG and every boot is much slower" +fi +log "accelerator: $accel" + +fetch() { + local url="$1" dest="$2" sha="$3" + if [ -f "$dest" ] && echo "$sha $dest" | sha256sum -c --status -; then + log "cached: $dest" + return 0 + fi + log "downloading $url" + mkdir -p "$(dirname "$dest")" + if ! curl -fsSL --retry 3 -o "$dest.tmp" "$url"; then + rm -f "$dest.tmp" + return 2 + fi + if ! echo "$sha $dest.tmp" | sha256sum -c --status -; then + log "checksum mismatch for $url (want $sha, got $(sha256sum "$dest.tmp" | cut -d' ' -f1))" + rm -f "$dest.tmp" + exit 1 + fi + mv "$dest.tmp" "$dest" +} + +kind_bin="$cache/kind-$KIND_VERSION-amd64" +fetch "https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64" "$kind_bin" "$KIND_SHA256_AMD64" || + skip "kind $KIND_VERSION is not cached and could not be downloaded" +chmod +x "$kind_bin" +kubectl_bin="$cache/kubectl-$KUBECTL_VERSION-amd64" +fetch "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl" "$kubectl_bin" "$KUBECTL_SHA256_AMD64" || + { log "kubectl $KUBECTL_VERSION could not be downloaded"; exit 1; } +chmod +x "$kubectl_bin" +cm_yaml="$cache/cert-manager-$CERT_MANAGER_VERSION.yaml" +fetch "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" "$cm_yaml" "$CERT_MANAGER_SHA256" || + { log "cert-manager $CERT_MANAGER_VERSION manifest could not be downloaded"; exit 1; } +contour_yaml="$cache/contour-$CONTOUR_VERSION.yaml" +fetch "https://raw.githubusercontent.com/projectcontour/contour/$CONTOUR_VERSION/examples/render/contour.yaml" "$contour_yaml" "$CONTOUR_SHA256" || + { log "Contour $CONTOUR_VERSION manifest could not be downloaded"; exit 1; } + +kind() { "$kind_bin" "$@"; } +kubectl() { "$kubectl_bin" --kubeconfig "$kubeconfig" "$@"; } + +# ---- teardown -------------------------------------------------------------- + +added_hosts=0 +# shellcheck disable=SC2329 # run by the EXIT trap below +cleanup() { + local rc=$? + if [ "$added_hosts" = 1 ]; then + log "removing the $hostname line from /etc/hosts" + sudo sed -i "/$hosts_marker\$/d" /etc/hosts || true + fi + docker rm -f cryptos-e2e-nginx >/dev/null 2>&1 || true + if [ "${KEEP_CLUSTER:-0}" = 1 ]; then + log "KEEP_CLUSTER=1: leaving cluster $cluster up (kubeconfig: $kubeconfig)" + else + kind delete cluster --name "$cluster" >/dev/null 2>&1 || true + rm -rf "$work" + fi + log "logs, summary and coverage: $out" + exit "$rc" +} +trap cleanup EXIT + +# ---- name resolution for the VMs -------------------------------------------- + +# A VM resolves names through QEMU's DNS proxy, which asks this host's +# resolver; systemd-resolved answers from /etc/hosts. The name points at the +# host as the VM sees it, so the node's http-01 fetch reaches the ingress on +# this host's loopback. Nothing on the host itself dials the name. +if ! grep -qE "^[^#]*[[:space:]]$hostname([[:space:]]|\$)" /etc/hosts; then + if ! sudo -n true 2>/dev/null; then + log "add this line to /etc/hosts and rerun (sudo is needed to add it for you):" + log " $guest_host_ip $hostname" + exit 1 + fi + echo "$guest_host_ip $hostname $hosts_marker" | sudo tee -a /etc/hosts >/dev/null + added_hosts=1 + log "added $guest_host_ip $hostname to /etc/hosts" +fi + +# ---- cluster --------------------------------------------------------------- + +started=$SECONDS +kind delete cluster --name "$cluster" >/dev/null 2>&1 || true +log "creating kind cluster $cluster ($KIND_NODE_IMAGE)" +kind create cluster --name "$cluster" --image "$KIND_NODE_IMAGE" \ + --config "$root/test/kind/kind-config.yaml" --kubeconfig "$kubeconfig" --wait 120s + +host_ip="$(docker inspect -f '{{with index .NetworkSettings.Networks "kind"}}{{.Gateway}}{{end}}' "$cluster-control-plane")" +[ -n "$host_ip" ] || { log "could not read the kind network gateway"; exit 1; } +log "pods reach this host at $host_ip" + +kubectl apply -f "$contour_yaml" >/dev/null +kubectl apply -f "$cm_yaml" >/dev/null +sed -e "s|WHOAMI_IMAGE|$WHOAMI_IMAGE|" -e "s|E2E_HOSTNAME|$hostname|g" "$root/test/kind/whoami.yaml" | kubectl apply -f - >/dev/null +kubectl -n projectcontour wait --for=condition=complete job --all --timeout=180s +kubectl -n projectcontour rollout status deployment/contour --timeout=180s +kubectl -n projectcontour rollout status daemonset/envoy --timeout=180s +kubectl -n cert-manager wait --for=condition=Available deployment --all --timeout=180s +kubectl -n cryptos-e2e rollout status deployment/whoami --timeout=180s + +envoy_ip="$(kubectl -n projectcontour get service envoy -o jsonpath='{.spec.clusterIP}')" +corefile="$(kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}')" +corefile="$(printf '%s\n' "$corefile" | awk -v ip="$envoy_ip" -v h="$hostname" ' + { print } + /^\.:53 \{/ { print " hosts {\n " ip " " h "\n fallthrough\n }" }')" +kubectl -n kube-system create configmap coredns --from-literal=Corefile="$corefile" --dry-run=client -o yaml | + kubectl apply -f - >/dev/null +kubectl -n kube-system rollout restart deployment/coredns >/dev/null +kubectl -n kube-system rollout status deployment/coredns --timeout=120s +log "cluster ready in $((SECONDS - started))s" + +docker pull -q "$NGINX_IMAGE" >/dev/null + +# ---- coverage-instrumented cryptosctl ---------------------------------------- + +cover_dir="$out/coverage" +rm -rf "$cover_dir" +mkdir -p "$cover_dir/cryptosctl" +cryptosctl="$work/cryptosctl" +( cd "$root" && CGO_ENABLED=0 go build -trimpath -cover -covermode=atomic -coverpkg=github.com/CryptOS-PKI/cryptos/... \ + -ldflags "$(bash build/ci/buildinfo.sh)" -o "$cryptosctl" ./cmd/cryptosctl ) + +# ---- the suite --------------------------------------------------------------- + +log "running TestImageSuite" +set +e +( + cd "$root" && + CRYPTOS_E2E_IMAGE=1 \ + E2E_IMAGE_UKI="$uki" \ + E2E_IMAGE_NEXT_UKI="${uki%.uki}-next.uki" \ + E2E_IMAGE_ACCEL="$accel" \ + E2E_IMAGE_OUT="$out" \ + E2E_IMAGE_COVERDIR="$cover_dir" \ + OVMF_CODE="$ovmf_code" OVMF_VARS="$ovmf_vars" \ + CRYPTOSCTL="$cryptosctl" \ + E2E_IMAGE_HOST_IP="$host_ip" \ + E2E_IMAGE_HOSTNAME="$hostname" \ + E2E_IMAGE_KUBECONFIG="$kubeconfig" \ + E2E_IMAGE_KUBECTL="$kubectl_bin" \ + E2E_IMAGE_NGINX="$NGINX_IMAGE" \ + go test -tags=integration -count=1 -v -timeout 35m -run '^TestImageSuite$' ./test/integration/ +) 2>&1 | tee "$out/suite.log" +rc=${PIPESTATUS[0]} +set -e + +# ---- coverage ------------------------------------------------------------------ + +bash "$here/coverage.sh" "$cover_dir" "$out" || log "coverage report failed (the suite result stands)" + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + [ -f "$out/summary.md" ] && cat "$out/summary.md" >>"$GITHUB_STEP_SUMMARY" + [ -f "$out/coverage-summary.md" ] && cat "$out/coverage-summary.md" >>"$GITHUB_STEP_SUMMARY" +fi +exit "$rc" diff --git a/test/image/versions.env b/test/image/versions.env new file mode 100644 index 0000000..f3c9a57 --- /dev/null +++ b/test/image/versions.env @@ -0,0 +1,7 @@ +# Pins for the full-image suite (test/image/run.sh, `task e2e:image`) on top +# of the kind, kubectl, cert-manager, Contour and whoami pins it reuses from +# test/kind/versions.env. Every image is pinned by digest; bump deliberately +# and re-verify against the upstream registry. + +# nginx serves an issued leaf with OCSP stapling (stable line). +NGINX_IMAGE=docker.io/library/nginx:1.30.5-alpine@sha256:0985e772fb9f729e6fa0980da05fca5d9c468e870eed43071545afa9d2e27d94 diff --git a/test/integration/image_suite_test.go b/test/integration/image_suite_test.go new file mode 100644 index 0000000..e0e07e2 --- /dev/null +++ b/test/integration/image_suite_test.go @@ -0,0 +1,1655 @@ +//go:build integration + +package integration + +/* +Copyright The CryptOS Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The full-image suite: the real CryptOS image, booted in QEMU with swtpm and +// OVMF as a Root and an Intermediate, driven over the mTLS API with cryptosctl +// the way an operator drives it. Each step is a named subtest whose result +// (pass, fail, or skip with the reason) goes into a summary table, and a step +// whose prerequisite failed is skipped rather than failing on top of it. +// +// test/image/run.sh (task e2e:image) stands up what the steps need around the +// VMs (a kind cluster with cert-manager, the nginx image, a +// coverage-instrumented cryptosctl) and sets the E2E_IMAGE_* variables below. +// Without CRYPTOS_E2E_IMAGE=1 the test skips. +// +// Networking: every VM gets its own QEMU user-mode network (10.0.0.0/24, the +// node at 10.0.0.10, the host at 10.0.0.1, QEMU's DNS proxy at 10.0.0.3) and +// host-forwards for the ports a step dials. User networking needs no root and +// no bridge on the runner, and each VM's network is isolated from the others. +// A guest connection to 10.0.0.1 lands on the host's loopback, which is where +// kind maps the cluster ingress, so the node's http-01 fetch reaches the +// cluster without any bypass; its DNS query for the name goes through QEMU's +// proxy to the host resolver, which run.sh points at 10.0.0.1. + +import ( + "bytes" + "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/crypto/ocsp" +) + +const ( + suiteRootCN = "CryptOS E2E Root" + suiteIntCN = "CryptOS E2E Issuing G1" + + suiteNodeAddr = "10.0.0.10" + + rootMgmtPort = "4443" + intMgmtPort = "4444" + spareMgmtPort = "4445" + intRevPort = "18080" + intACMEPort = "18555" + intESTPort = "18443" + acmeFrontPort = "8443" + nginxPort = "9443" + + nginxName = "nginx.cryptos.test" + nginxContainer = "cryptos-e2e-nginx" + estUser = "e2e-device" + estClientName = "est-client.cryptos.test" + acmeEABKeyID = "e2e-image" + acmeLeafProfile = "acme-leaf" + + kindNS = "cryptos-e2e" + kindCert = "whoami" + kindSecret = "whoami-tls" +) + +// suiteEnv is what run.sh hands the suite. +type suiteEnv struct { + qemu, swtpm, ovmfCode, ovmfVars string + uki, nextUKI, accel, out, coverDir string + cryptosctl string + hostIP, hostname string + kubeconfig, kubectl, nginx string +} + +func loadSuiteEnv(t *testing.T) suiteEnv { + t.Helper() + if os.Getenv("CRYPTOS_E2E_IMAGE") != "1" { + t.Skip("full-image suite: not requested; run `task e2e:image` on a Linux host") + } + get := func(k string) string { return strings.TrimSpace(os.Getenv(k)) } + e := suiteEnv{ + qemu: firstNonEmpty(get("QEMU"), "qemu-system-x86_64"), + swtpm: firstNonEmpty(get("SWTPM"), "swtpm"), + ovmfCode: get("OVMF_CODE"), + ovmfVars: get("OVMF_VARS"), + uki: get("E2E_IMAGE_UKI"), + nextUKI: get("E2E_IMAGE_NEXT_UKI"), + accel: firstNonEmpty(get("E2E_IMAGE_ACCEL"), "kvm:tcg"), + out: get("E2E_IMAGE_OUT"), + coverDir: get("E2E_IMAGE_COVERDIR"), + cryptosctl: get("CRYPTOSCTL"), + hostIP: get("E2E_IMAGE_HOST_IP"), + hostname: get("E2E_IMAGE_HOSTNAME"), + kubeconfig: get("E2E_IMAGE_KUBECONFIG"), + kubectl: firstNonEmpty(get("E2E_IMAGE_KUBECTL"), "kubectl"), + nginx: get("E2E_IMAGE_NGINX"), + } + for k, v := range map[string]string{ + "OVMF_CODE": e.ovmfCode, "OVMF_VARS": e.ovmfVars, "E2E_IMAGE_UKI": e.uki, + "E2E_IMAGE_OUT": e.out, "CRYPTOSCTL": e.cryptosctl, "E2E_IMAGE_HOST_IP": e.hostIP, + "E2E_IMAGE_HOSTNAME": e.hostname, "E2E_IMAGE_KUBECONFIG": e.kubeconfig, "E2E_IMAGE_NGINX": e.nginx, + } { + if v == "" { + t.Fatalf("full-image suite: %s must be set (run it through test/image/run.sh)", k) + } + } + if net.ParseIP(e.hostIP) == nil { + t.Fatalf("full-image suite: E2E_IMAGE_HOST_IP %q is not an IP", e.hostIP) + } + return e +} + +// ---- the step runner and its summary ---------------------------------------- + +type stepResult struct { + name, status, note string + took time.Duration +} + +type suite struct { + t *testing.T + env suiteEnv + mu sync.Mutex + results []stepResult + passed map[string]bool + notes []string +} + +// step runs fn as the subtest name, unless a step in needs did not pass, in +// which case it is skipped with that reason. +func (s *suite) step(name string, needs []string, fn func(t *testing.T)) { + s.t.Run(name, func(t *testing.T) { + started := time.Now() + defer func() { + res := stepResult{name: name, took: time.Since(started)} + switch { + case t.Failed(): + res.status = "fail" + case t.Skipped(): + res.status = "skip" + default: + res.status = "pass" + } + s.mu.Lock() + if res.status == "pass" { + s.passed[name] = true + } + if n, ok := stepNotes.LoadAndDelete(t.Name()); ok { + res.note = n.(string) + } + s.results = append(s.results, res) + s.mu.Unlock() + }() + for _, n := range needs { + if !s.passed[n] { + note(t, "needs "+n) + t.Skipf("skipped: needs %s, which did not pass", n) + } + } + fn(t) + }) +} + +// skipStep records a step the suite cannot run yet, with the reason. +func (s *suite) skipStep(name, reason string) { + s.step(name, nil, func(t *testing.T) { + note(t, reason) + t.Skip(reason) + }) +} + +var stepNotes sync.Map + +// note sets the summary note for the running step. +func note(t *testing.T, msg string) { + stepNotes.Store(t.Name(), msg) +} + +func (s *suite) writeSummary() { + var b strings.Builder + b.WriteString("## Full-image suite\n\n") + b.WriteString("Booted image: `" + filepath.Base(s.env.uki) + "`, accelerator `" + s.env.accel + "`.\n\n") + b.WriteString("| # | Step | Result | Time | Notes |\n|---|---|---|---|---|\n") + counts := map[string]int{} + for i, r := range s.results { + icon := map[string]string{"pass": "pass", "fail": "**FAIL**", "skip": "skip"}[r.status] + counts[r.status]++ + fmt.Fprintf(&b, "| %d | %s | %s | %s | %s |\n", i+1, r.name, icon, r.took.Round(time.Second), + strings.ReplaceAll(r.note, "|", "\\|")) + } + fmt.Fprintf(&b, "\n%d passed, %d failed, %d skipped.\n", counts["pass"], counts["fail"], counts["skip"]) + if len(s.notes) > 0 { + b.WriteString("\n") + for _, n := range s.notes { + b.WriteString("- " + n + "\n") + } + } + b.WriteString("\n") + if err := os.WriteFile(filepath.Join(s.env.out, "summary.md"), []byte(b.String()), 0o644); err != nil { + s.t.Logf("write summary: %v", err) + } + s.t.Log("\n" + b.String()) +} + +func (s *suite) timing(format string, args ...any) { + s.mu.Lock() + defer s.mu.Unlock() + s.notes = append(s.notes, fmt.Sprintf(format, args...)) +} + +// ---- VMs -------------------------------------------------------------------- + +// vm is one booted CryptOS node: its disk, TPM, serial line and forwards. +type vm struct { + s *suite + name string + dir string + uuid string + mgmtPort string + forwards []string + withTPM bool + disk string + cover string + vars string + cmd *exec.Cmd + keys io.WriteCloser + serial *serialLog + trust string + admin admin + stopped bool +} + +type admin struct{ cert, key string } + +func (s *suite) newVM(t *testing.T, name, uuid, mgmtPort string, forwards []string, machineYAML string, a admin) *vm { + t.Helper() + dir := filepath.Join(s.env.out, "vm-"+name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("vm dir: %v", err) + } + cfg := filepath.Join(dir, "machine.yaml") + writeFile(t, cfg, []byte(machineYAML)) + v := &vm{s: s, name: name, dir: dir, uuid: uuid, mgmtPort: mgmtPort, forwards: forwards, withTPM: true, admin: a} + v.disk = makeInstalledDisk(t, dir, cfg) + // The installer's layout: the UKI at the removable-media path on the + // disk's own ESP, which OVMF boots and the image upgrader reads back. + off := v.disk + "@@" + espPartitionOffset(t, v.disk) + if out, err := exec.Command("mmd", "-i", off, "::/EFI/BOOT").CombinedOutput(); err != nil { + t.Fatalf("mmd EFI/BOOT: %v\n%s", err, out) + } + if out, err := exec.Command("mcopy", "-i", off, s.env.uki, "::/EFI/BOOT/BOOTX64.EFI").CombinedOutput(); err != nil { + t.Fatalf("mcopy the UKI onto the ESP: %v\n%s", err, out) + } + v.cover = filepath.Join(dir, "cover.img") + if err := exec.Command("truncate", "-s", "128M", v.cover).Run(); err != nil { + t.Fatalf("create the coverage disk: %v", err) + } + if out, err := exec.Command("mformat", "-i", v.cover, "-v", "COVER", "::").CombinedOutput(); err != nil { + t.Fatalf("mformat the coverage disk: %v\n%s", err, out) + } + v.vars = filepath.Join(dir, "OVMF_VARS.fd") + copyFile(t, s.env.ovmfVars, v.vars) + return v +} + +func (v *vm) boot(t *testing.T) { + t.Helper() + e := v.s.env + args := []string{ + "-machine", "q35,accel=" + e.accel, "-smp", "2", "-m", "2048", + "-uuid", v.uuid, + "-display", "none", "-monitor", "none", "-serial", "stdio", + "-drive", "if=pflash,format=raw,unit=0,readonly=on,file=" + e.ovmfCode, + "-drive", "if=pflash,format=raw,unit=1,file=" + v.vars, + "-drive", "if=none,id=disk,format=raw,file=" + v.disk, + "-device", "virtio-blk-pci,drive=disk,bootindex=1", + "-drive", "if=none,id=cover,format=raw,file=" + v.cover, + "-device", "virtio-blk-pci,drive=cover,serial=cryptos-cover", + "-netdev", "user,id=n0,net=10.0.0.0/24,host=10.0.0.1,dns=10.0.0.3," + strings.Join(append([]string{ + "hostfwd=tcp:127.0.0.1:" + v.mgmtPort + "-" + suiteNodeAddr + ":443"}, v.forwards...), ","), + "-device", "virtio-net-pci,netdev=n0", + } + if strings.HasPrefix(e.accel, "kvm") { + args = append(args, "-cpu", "host") + } + if v.withTPM { + sock := v.startTPM(t) + args = append(args, + "-chardev", "socket,id=chrtpm,path="+sock, + "-tpmdev", "emulator,id=tpm0,chardev=chrtpm", + "-device", "tpm-tis,tpmdev=tpm0") + } + logf, err := os.Create(filepath.Join(v.dir, "serial.log")) + if err != nil { + t.Fatalf("serial log: %v", err) + } + v.serial = &serialLog{f: logf} + keysR, keysW := io.Pipe() + v.keys = keysW + v.cmd = exec.Command(e.qemu, args...) + v.cmd.Stdin = keysR + v.cmd.Stdout, v.cmd.Stderr = v.serial, v.serial + started := time.Now() + if err := v.cmd.Start(); err != nil { + t.Fatalf("start qemu for %s: %v", v.name, err) + } + v.s.t.Cleanup(func() { v.stop() }) + v.waitUp(t, -1, started, "first boot") +} + +// startTPM starts the VM's swtpm, tied to the whole suite rather than to the +// step that booted the VM, and waits for its socket. +func (v *vm) startTPM(t *testing.T) string { + t.Helper() + state := filepath.Join(v.dir, "swtpm") + if err := os.MkdirAll(state, 0o700); err != nil { + t.Fatalf("swtpm state dir: %v", err) + } + sock := filepath.Join(state, "sock") + cmd := exec.Command(v.s.env.swtpm, "socket", "--tpm2", "--tpmstate", "dir="+state, "--ctrl", "type=unixio,path="+sock) + if err := cmd.Start(); err != nil { + t.Fatalf("start swtpm: %v", err) + } + v.s.t.Cleanup(func() { _ = cmd.Process.Kill(); _, _ = cmd.Process.Wait() }) + deadline := time.Now().Add(10 * time.Second) + for { + if _, err := os.Stat(sock); err == nil { + return sock + } + if time.Now().After(deadline) { + t.Fatalf("swtpm did not create %s", sock) + } + time.Sleep(100 * time.Millisecond) + } +} + +// waitUp waits for the management listener after a boot that began at +// started and re-pins the node's trust. After a reboot, the node is back only +// once its status reports a boot count above bootsBefore: the old boot keeps +// answering for a moment after the request, and its server certificate can +// change without a reboot (a re-certified CA re-mints it), so neither a +// listener that answers nor a new certificate proves a new boot. A first boot +// passes bootsBefore < 0 and waits for the listener alone. The serial log is +// not used for this: init logs through /dev/kmsg, which the kernel +// rate-limits, so a line can be lost. +func (v *vm) waitUp(t *testing.T, bootsBefore int, started time.Time, what string) { + t.Helper() + limit := 3 * time.Minute + if !strings.HasPrefix(v.s.env.accel, "kvm") { + limit = 15 * time.Minute + } + addr := "127.0.0.1:" + v.mgmtPort + deadline := time.Now().Add(limit) + var chain []*x509.Certificate + for { + chain = probeServerChain(addr) + if chain != nil { + v.pin(t, chain) + if bootsBefore < 0 { + break + } + if out, err := v.ctl("", "status"); err == nil && bootCount(out) > bootsBefore { + break + } + } + if time.Now().After(deadline) { + t.Fatalf("%s: %s did not bring the management API up within %s:\n%s", v.name, what, limit, lastLines(v.serial.String(), 60)) + } + time.Sleep(time.Second) + } + took := time.Since(started).Round(time.Second) + t.Logf("%s: %s up in %s", v.name, what, took) + v.s.timing("%s %s: management API up %s after %s", v.name, what, took, map[bool]string{true: "QEMU start", false: "the reboot request"}[bootsBefore < 0]) +} + +// pin trusts the last certificate the listener presented: the self-signed +// leaf before the node has a CA, and the root its CA-signed leaf chains to +// after. +func (v *vm) pin(t *testing.T, chain []*x509.Certificate) { + t.Helper() + v.trust = filepath.Join(v.dir, "trust.crt") + writeFile(t, v.trust, certPEM(chain[len(chain)-1])) +} + +// trustCA waits for the listener to present a leaf signed by the node's new +// CA, which it switches to without a restart once the CA is committed, and +// trusts the root that leaf chains to. +func (v *vm) trustCA(t *testing.T) { + t.Helper() + addr := "127.0.0.1:" + v.mgmtPort + deadline := time.Now().Add(2 * time.Minute) + for { + chain := probeServerChain(addr) + if len(chain) > 1 && chain[len(chain)-1].IsCA && chain[0].CheckSignatureFrom(chain[1]) == nil { + v.pin(t, chain) + return + } + if time.Now().After(deadline) { + t.Fatalf("%s: the management listener did not switch to a CA-signed certificate within 2m", v.name) + } + time.Sleep(time.Second) + } +} + +// probeServerChain returns the certificates the listener at addr presents, +// leaf first, or nil while it does not answer. +func probeServerChain(addr string) []*x509.Certificate { + conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 2 * time.Second}, "tcp", addr, + &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS13}) //nolint:gosec // pinning grab + if err != nil { + return nil + } + defer func() { _ = conn.Close() }() + certs := conn.ConnectionState().PeerCertificates + if len(certs) == 0 { + return nil + } + return certs +} + +func certPEM(c *x509.Certificate) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.Raw}) +} + +// ctl runs cryptosctl against the node as the bootstrap admin. The binary is +// coverage-instrumented, so each run writes its counters for the report. +func (v *vm) ctl(stdin string, args ...string) (string, error) { + full := append([]string{ + "--endpoint", "127.0.0.1:" + v.mgmtPort, + "--identity", v.admin.cert, "--identity-key", v.admin.key, + "--trust", v.trust, "--server-name", suiteNodeAddr, + }, args...) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, v.s.env.cryptosctl, full...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + cmd.Env = os.Environ() + if v.s.env.coverDir != "" { + cmd.Env = append(cmd.Env, "GOCOVERDIR="+filepath.Join(v.s.env.coverDir, "cryptosctl")) + } + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + out := stdout.String() + if stderr.Len() > 0 { + out += stderr.String() + } + return out, err +} + +func (v *vm) mustCtl(t *testing.T, args ...string) string { + t.Helper() + out, err := v.ctl("", args...) + if err != nil { + t.Fatalf("%s: cryptosctl %s: %v\n%s", v.name, strings.Join(args, " "), err, out) + } + return out +} + +func (v *vm) status(t *testing.T) string { + t.Helper() + return v.mustCtl(t, "status") +} + +// reboot asks the node for a clean reboot over the API and waits for it to +// come back. +func (v *vm) reboot(t *testing.T, cn string) { + t.Helper() + bootsBefore := bootCount(v.status(t)) + started := time.Now() + out := v.mustCtl(t, "reboot", "--confirm", cn) + if !strings.Contains(out, "reboot accepted") { + t.Fatalf("%s: reboot: %s", v.name, out) + } + v.waitUp(t, bootsBefore, started, "reboot") + if after := bootCount(v.status(t)); after != bootsBefore+1 { + t.Fatalf("%s: boot count %d after the reboot, want %d", v.name, after, bootsBefore+1) + } +} + +// stop leaves the coverage flusher one more round, stops QEMU and copies the +// counters the node wrote off its coverage disk. +func (v *vm) stop() { + if v.stopped || v.cmd == nil || v.cmd.Process == nil { + return + } + v.stopped = true + time.Sleep(5 * time.Second) + _ = v.keys.Close() + _ = v.cmd.Process.Kill() + _, _ = v.cmd.Process.Wait() + if v.s.env.coverDir == "" { + return + } + dest := filepath.Join(v.s.env.coverDir, "vm-"+v.name) + _ = os.MkdirAll(dest, 0o755) + if out, err := exec.Command("mcopy", "-n", "-i", v.cover, "::/*", dest).CombinedOutput(); err != nil { + v.s.t.Logf("%s: copy coverage counters off the VM: %v\n%s", v.name, err, out) + } + _ = os.Remove(filepath.Join(dest, "partial")) +} + +func bootCount(status string) int { + var n int + for _, l := range strings.Split(status, "\n") { + if strings.HasPrefix(l, "Boot count:") { + _, _ = fmt.Sscanf(strings.TrimSpace(strings.TrimPrefix(l, "Boot count:")), "%d", &n) + } + } + return n +} + +func (s *serialLog) len() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.Len() +} + +func (s *serialLog) since(offset int) string { + s.mu.Lock() + defer s.mu.Unlock() + b := s.buf.Bytes() + if offset > len(b) { + return "" + } + return string(b[offset:]) +} + +// ---- machine configs ---------------------------------------------------------- + +func indentBlock(s, prefix string) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + for i, l := range lines { + lines[i] = prefix + l + } + return strings.Join(lines, "\n") +} + +// rootYAML is the Root: software CA key (nodeid state key), so its key can be +// escrowed, and one CA profile to sign the Intermediate. +func rootYAML(adminPEM, name string) string { + return fmt.Sprintf(`apiVersion: cryptos.dev/v1alpha1 +kind: MachineConfig +metadata: {name: %s} +role: {kind: root} +network: {interface: eth0, address: %s/24, gateway: 10.0.0.1, nameservers: [10.0.0.3]} +state_key: {mode: nodeid} +bootstrap: + admin_cert_pem: | +%s +pki: + root_key_alg: ECDSA-P384 + root_subject: {common_name: %q, organization: "CryptOS E2E", country: "US"} + root_validity_years: 10 + profiles: + - name: sub-ca + key_alg: ECDSA-P384 + subject: {common_name: sub-ca} + validity_days: 1825 + basic_constraints: {is_ca: true, path_len: 0} + key_usage: [digital_signature, cert_sign, crl_sign] +`, name, suiteNodeAddr, indentBlock(adminPEM, " "), suiteRootCN) +} + +type intProtocols struct { + acme, est bool + eabKeyB64 string + estPassSHA256 string +} + +// intYAML is the Intermediate: TPM-held CA key, the Root pinned as its +// parent, revocation published on the runner-reachable forward, and the +// profiles the steps issue under. ACME and EST are rendered only when on. +func intYAML(adminPEM, rootPEM, hostIP string, p intProtocols) string { + var b strings.Builder + fmt.Fprintf(&b, `apiVersion: cryptos.dev/v1alpha1 +kind: MachineConfig +metadata: {name: e2e-intermediate} +role: {kind: intermediate} +network: {interface: eth0, address: %s/24, gateway: 10.0.0.1, nameservers: [10.0.0.3]} +state_key: {mode: tpm} +bootstrap: + admin_cert_pem: | +%s +pki: + root_key_alg: ECDSA-P384 + root_subject: {common_name: %q, organization: "CryptOS E2E", country: "US"} + root_validity_years: 5 + path_len_constraint: 0 + revocation_base_url: http://%s:%s + parent: + ca_cert_pem: | +%s + profiles: + - name: tls-server + key_alg: ECDSA-P384 + subject: {common_name: tls-server} + validity_days: 30 + basic_constraints: {is_ca: false} + key_usage: [digital_signature] + ext_key_usage: [server_auth] + allow_request_sans: true + - name: %s + key_alg: ECDSA-P384 + subject: {common_name: acme} + validity_days: 30 + basic_constraints: {is_ca: false} + key_usage: [digital_signature] + ext_key_usage: [server_auth] + - name: acme-front + key_alg: ECDSA-P384 + subject: {common_name: cryptos-acme-front} + validity_days: 7 + basic_constraints: {is_ca: false} + key_usage: [digital_signature] + ext_key_usage: [server_auth] + sans: {ip: [%q]} + - name: est-client + key_alg: ECDSA-P384 + subject: {common_name: est-client} + validity_days: 30 + basic_constraints: {is_ca: false} + key_usage: [digital_signature] + ext_key_usage: [client_auth, server_auth] +`, suiteNodeAddr, indentBlock(adminPEM, " "), suiteIntCN, hostIP, intRevPort, + indentBlock(rootPEM, " "), acmeLeafProfile, hostIP) + if p.acme { + fmt.Fprintf(&b, ` acme: + base_url: https://%s:%s/acme + profile: %s + allowed_identifier_suffixes: [cryptos.test] + external_account_keys: + - key_id: %s + hmac_key_base64: %s +`, hostIP, acmeFrontPort, acmeLeafProfile, acmeEABKeyID, p.eabKeyB64) + } + if p.est { + fmt.Fprintf(&b, ` est: + hostnames: ["127.0.0.1"] + profile: est-client + allowed_identifier_suffixes: [cryptos.test] + enroll_credentials: + - username: %s + password_sha256: %s +`, estUser, p.estPassSHA256) + } + return b.String() +} + +// ---- keys, CSRs and certificates ------------------------------------------------ + +func newCSR(t *testing.T, key crypto.Signer, cn string, dns ...string) []byte { + t.Helper() + der, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{ + Subject: pkix.Name{CommonName: cn}, + DNSNames: dns, + }, key) + if err != nil { + t.Fatalf("CSR: %v", err) + } + return der +} + +func pemBlock(typ string, der []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: typ, Bytes: der}) +} + +func keyPEM(t *testing.T, key crypto.Signer) []byte { + t.Helper() + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + return pemBlock("PRIVATE KEY", der) +} + +func p384(t *testing.T) *ecdsa.PrivateKey { + t.Helper() + k, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + t.Fatalf("P-384 key: %v", err) + } + return k +} + +func rsaKey(t *testing.T, bits int) *rsa.PrivateKey { + t.Helper() + k, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + t.Fatalf("RSA-%d key: %v", bits, err) + } + return k +} + +func parsePEMCerts(t *testing.T, data []byte) []*x509.Certificate { + t.Helper() + var out []*x509.Certificate + for rest := data; ; { + var b *pem.Block + b, rest = pem.Decode(rest) + if b == nil { + break + } + if b.Type != "CERTIFICATE" { + continue + } + c, err := x509.ParseCertificate(b.Bytes) + if err != nil { + t.Fatalf("parse certificate: %v", err) + } + out = append(out, c) + } + if len(out) == 0 { + t.Fatalf("no certificate in:\n%s", data) + } + return out +} + +func verifyTo(t *testing.T, leaf *x509.Certificate, root *x509.Certificate, inter []*x509.Certificate, dns string, eku x509.ExtKeyUsage) { + t.Helper() + roots := x509.NewCertPool() + roots.AddCert(root) + ip := x509.NewCertPool() + for _, c := range inter { + ip.AddCert(c) + } + if _, err := leaf.Verify(x509.VerifyOptions{DNSName: dns, Roots: roots, Intermediates: ip, KeyUsages: []x509.ExtKeyUsage{eku}}); err != nil { + t.Fatalf("certificate %s (serial %s) does not verify to the Root: %v", leaf.Subject, leaf.SerialNumber.Text(16), err) + } +} + +func run(t *testing.T, stdin string, name string, args ...string) (string, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, name, args...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + out, err := cmd.CombinedOutput() + return string(out), err +} + +func mustRun(t *testing.T, name string, args ...string) string { + t.Helper() + out, err := run(t, "", name, args...) + if err != nil { + t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out) + } + return out +} + +func eventually(t *testing.T, what string, timeout time.Duration, check func() (bool, string)) { + t.Helper() + deadline := time.Now().Add(timeout) + var last string + for { + ok, state := check() + if ok { + return + } + last = state + if time.Now().After(deadline) { + t.Fatalf("%s: not reached within %s; last: %s", what, timeout, last) + } + time.Sleep(3 * time.Second) + } +} + +// ---- the suite ------------------------------------------------------------------- + +// state is what one step hands the next. +type suiteState struct { + admin admin + adminPEM string + root *vm + inter *vm + spare *vm + rootCert *x509.Certificate + rootPEM string + intCert *x509.Certificate + intPEM string + eabKeyB64 string + estPass string + nginxDir string + nginxLeaf *x509.Certificate + backup string + backupPwd string +} + +func TestImageSuite(t *testing.T) { + e := loadSuiteEnv(t) + s := &suite{t: t, env: e, passed: map[string]bool{}} + t.Cleanup(s.writeSummary) + st := &suiteState{} + + certPEM, adminKeyPEM, _ := generateBootstrapAdmin(t) + st.admin = admin{cert: filepath.Join(e.out, "admin.crt"), key: filepath.Join(e.out, "admin.key")} + writeFile(t, st.admin.cert, certPEM) + writeFile(t, st.admin.key, adminKeyPEM) + st.adminPEM = string(certPEM) + + eab := make([]byte, 32) + _, _ = rand.Read(eab) + st.eabKeyB64 = base64.RawURLEncoding.EncodeToString(eab) + st.estPass = "est-" + hex.EncodeToString(eab[:8]) + + s.step("root: boot and first-boot ceremony", nil, func(t *testing.T) { stepRootCeremony(t, s, st) }) + s.step("intermediate: boot, sign-subordinate on the Root, submit", []string{"root: boot and first-boot ceremony"}, + func(t *testing.T) { stepIntermediate(t, s, st) }) + const hier = "intermediate: boot, sign-subordinate on the Root, submit" + s.step("leaf issuance: ECDSA P-384 and RSA 3072 issue, RSA 2048 refused", []string{hier}, + func(t *testing.T) { stepLeaves(t, s, st) }) + s.step("nginx serves the leaf with a good OCSP staple", []string{hier}, + func(t *testing.T) { stepNginx(t, s, st) }) + s.step("revocation: OCSP and the CRL say revoked, nginx stops stapling good", []string{"nginx serves the leaf with a good OCSP staple"}, + func(t *testing.T) { stepRevocation(t, s, st) }) + s.step("protocol switch: ACME and EST on, reboot pending, reboot, running", []string{hier}, + func(t *testing.T) { stepProtocolsOn(t, s, st) }) + const protoOn = "protocol switch: ACME and EST on, reboot pending, reboot, running" + s.step("ACME: cert-manager in kind gets and renews a certificate over http-01", []string{protoOn}, + func(t *testing.T) { stepACME(t, s, st) }) + s.step("EST: simpleenroll and simplereenroll with curl", []string{protoOn}, + func(t *testing.T) { stepEST(t, s, st) }) + s.step("re-certify the Intermediate on the same key", []string{hier}, + func(t *testing.T) { stepRecertify(t, s, st) }) + s.step("escrow: export refused on the TPM node, short passphrase refused, export from the Root", []string{hier}, + func(t *testing.T) { stepEscrowExport(t, s, st) }) + s.step("escrow: import the Root backup onto a fresh node", []string{"escrow: export refused on the TPM node, short passphrase refused, export from the Root"}, + func(t *testing.T) { stepEscrowImport(t, s, st) }) + s.skipStep("audit: the audit chain verifies", + "no RPC or cryptosctl command verifies the audit chain yet; audit.VerifyChain runs only in-process, and the log sits on the encrypted state partition") + s.skipStep("SNTP: the clock line shows synced against a local NTP server", + "the suite runs no NTP server for the node to sync against yet") + s.skipStep("SCEP: sscep enrols", + "the suite runs no SCEP client against the node yet") + s.step("image upgrade in place on the software-key Root, then it still signs", []string{hier}, + func(t *testing.T) { stepUpgrade(t, s, st, st.root, suiteRootCN) }) + s.step("image upgrade in place on the TPM Intermediate, then it still issues", []string{hier}, + func(t *testing.T) { stepUpgrade(t, s, st, st.inter, suiteIntCN) }) + s.step("protocol switch: ACME off, reboot pending, reboot, cleared", []string{protoOn}, + func(t *testing.T) { stepProtocolsOff(t, s, st) }) + s.step("console reset over the serial line, on a throwaway node", []string{"escrow: import the Root backup onto a fresh node"}, + func(t *testing.T) { stepConsoleReset(t, s, st) }) + + for _, v := range []*vm{st.spare, st.inter, st.root} { + if v != nil { + v.stop() + } + } +} + +func stepRootCeremony(t *testing.T, s *suite, st *suiteState) { + st.root = s.newVM(t, "root", "564d0001-0000-0000-0000-00000000e2e1", rootMgmtPort, nil, rootYAML(st.adminPEM, "e2e-root"), st.admin) + st.root.boot(t) + out := st.root.mustCtl(t, "ceremony", "start", "--config", filepath.Join(st.root.dir, "machine.yaml")) + for _, want := range []string{"KEY_CREATED", "CERT_SIGNED", "MANIFEST_WRITTEN", "ADMIN_ROTATED", "COMPLETE"} { + if !strings.Contains(out, want) { + t.Fatalf("ceremony output missing %q:\n%s", want, out) + } + } + st.root.trustCA(t) + st.rootPEM = st.root.mustCtl(t, "identity", "show", "-o", "pem") + st.rootCert = parsePEMCerts(t, []byte(st.rootPEM))[0] + writeFile(t, filepath.Join(s.env.out, "root.pem"), []byte(st.rootPEM)) + if out := st.root.mustCtl(t, "identity", "validate"); !strings.Contains(out, "OK") { + t.Fatalf("identity validate: %s", out) + } + status := st.root.status(t) + for _, want := range []string{"Role: ROOT", "Identity: ESTABLISHED", "TPM: UNAVAILABLE"} { + if !strings.Contains(status, want) { + t.Fatalf("root status missing %q:\n%s", want, status) + } + } + if st.rootCert.Subject.CommonName != suiteRootCN || !st.rootCert.IsCA { + t.Fatalf("root certificate: CN %q CA %t", st.rootCert.Subject.CommonName, st.rootCert.IsCA) + } + note(t, "software (nodeid) Root, ECDSA P-384") +} + +func stepIntermediate(t *testing.T, s *suite, st *suiteState) { + cfg := intYAML(st.adminPEM, st.rootPEM, s.env.hostIP, intProtocols{}) + st.inter = s.newVM(t, "intermediate", "564d0002-0000-0000-0000-00000000e2e2", intMgmtPort, []string{ + "hostfwd=tcp:" + s.env.hostIP + ":" + intRevPort + "-" + suiteNodeAddr + ":80", + "hostfwd=tcp:127.0.0.1:" + intACMEPort + "-" + suiteNodeAddr + ":8555", + "hostfwd=tcp:127.0.0.1:" + intESTPort + "-" + suiteNodeAddr + ":8443", + }, cfg, st.admin) + st.inter.boot(t) + if status := st.inter.status(t); !strings.Contains(status, "Identity: AWAITING_CERT") || !strings.Contains(status, "TPM: OK") { + t.Fatalf("intermediate before signing: want AWAITING_CERT with the TPM OK:\n%s", status) + } + csr := st.inter.mustCtl(t, "ca", "get-subordinate-csr") + csrPath := filepath.Join(st.inter.dir, "subordinate.csr") + writeFile(t, csrPath, []byte(csr)) + chain := st.root.mustCtl(t, "ca", "sign-subordinate", "--csr", csrPath, "--profile", "sub-ca") + chainPath := filepath.Join(st.inter.dir, "subordinate-chain.pem") + writeFile(t, chainPath, []byte(chain)) + st.inter.mustCtl(t, "ca", "submit-subordinate-cert", "--chain", chainPath) + st.inter.trustCA(t) + + st.intPEM = st.inter.mustCtl(t, "identity", "show", "-o", "pem") + certs := parsePEMCerts(t, []byte(st.intPEM)) + st.intCert = certs[0] + writeFile(t, filepath.Join(s.env.out, "intermediate.pem"), pemBlock("CERTIFICATE", st.intCert.Raw)) + roots := x509.NewCertPool() + roots.AddCert(st.rootCert) + if _, err := st.intCert.Verify(x509.VerifyOptions{Roots: roots, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}}); err != nil { + t.Fatalf("intermediate does not verify to the Root: %v", err) + } + if !st.intCert.IsCA || st.intCert.MaxPathLen != 0 || !st.intCert.MaxPathLenZero { + t.Fatalf("intermediate: CA %t pathlen %d (zero %t), want a CA with pathlen 0", st.intCert.IsCA, st.intCert.MaxPathLen, st.intCert.MaxPathLenZero) + } + if status := st.inter.status(t); !strings.Contains(status, "Identity: ESTABLISHED") { + t.Fatalf("intermediate after submit: not ESTABLISHED:\n%s", status) + } + note(t, "TPM-held Intermediate key, chain verifies to the Root") +} + +func stepLeaves(t *testing.T, s *suite, st *suiteState) { + dir := filepath.Join(s.env.out, "leaves") + _ = os.MkdirAll(dir, 0o755) + issue := func(name string, key crypto.Signer) (string, error) { + csr := filepath.Join(dir, name+".csr") + writeFile(t, csr, pemBlock("CERTIFICATE REQUEST", newCSR(t, key, name+".cryptos.test"))) + return st.inter.ctl("", "ca", "issue-leaf", "--csr", csr, "--profile", "tls-server", "--dns", name+".cryptos.test") + } + for _, c := range []struct { + name string + key crypto.Signer + }{{"leaf-p384", p384(t)}, {"leaf-rsa3072", rsaKey(t, 3072)}} { + out, err := issue(c.name, c.key) + if err != nil { + t.Fatalf("issue-leaf %s: %v\n%s", c.name, err, out) + } + leaf := parsePEMCerts(t, []byte(out))[0] + verifyTo(t, leaf, st.rootCert, []*x509.Certificate{st.intCert}, c.name+".cryptos.test", x509.ExtKeyUsageServerAuth) + if !listsSerial(t, st.inter, leaf.SerialNumber, "tls-server") { + t.Fatalf("ca list-issued does not show %s serial %s", c.name, leaf.SerialNumber.Text(16)) + } + } + out, err := issue("leaf-rsa2048", rsaKey(t, 2048)) + if err == nil { + t.Fatalf("an RSA 2048 CSR was issued, want it refused:\n%s", out) + } + if !strings.Contains(out, "at least 3072 bits") { + t.Fatalf("RSA 2048 refused with an unexpected error:\n%s", out) + } + note(t, "RSA 2048 refused: "+strings.TrimSpace(lastLines(out, 1))) +} + +// listsSerial reports whether `ca list-issued` on v shows serial under profile. +func listsSerial(t *testing.T, v *vm, serial *big.Int, profile string) bool { + t.Helper() + out := v.mustCtl(t, "ca", "list-issued") + want := strings.ToLower(serial.Text(16)) + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + if len(f) > 0 && strings.ToLower(f[0]) == want { + return profile == "" || strings.Contains(line, profile) + } + } + return false +} + +func stepNginx(t *testing.T, s *suite, st *suiteState) { + dir := filepath.Join(s.env.out, "nginx") + _ = os.MkdirAll(dir, 0o755) + key := p384(t) + csr := filepath.Join(dir, "leaf.csr") + writeFile(t, csr, pemBlock("CERTIFICATE REQUEST", newCSR(t, key, nginxName))) + leafPEM := st.inter.mustCtl(t, "ca", "issue-leaf", "--csr", csr, "--profile", "tls-server", "--dns", nginxName) + st.nginxLeaf = parsePEMCerts(t, []byte(leafPEM))[0] + if len(st.nginxLeaf.OCSPServer) == 0 { + t.Fatalf("leaf carries no OCSP pointer (AIA); revocation_base_url is set on the Intermediate") + } + t.Logf("leaf serial %s, OCSP %v, CRL %v", st.nginxLeaf.SerialNumber.Text(16), st.nginxLeaf.OCSPServer, st.nginxLeaf.CRLDistributionPoints) + + intPEM := pemBlock("CERTIFICATE", st.intCert.Raw) + writeFile(t, filepath.Join(dir, "fullchain.pem"), append(pemBlock("CERTIFICATE", st.nginxLeaf.Raw), intPEM...)) + writeFile(t, filepath.Join(dir, "leaf.pem"), pemBlock("CERTIFICATE", st.nginxLeaf.Raw)) + writeFile(t, filepath.Join(dir, "leaf.key"), keyPEM(t, key)) + writeFile(t, filepath.Join(dir, "intermediate.pem"), intPEM) + writeFile(t, filepath.Join(dir, "root.pem"), []byte(st.rootPEM)) + writeFile(t, filepath.Join(dir, "trusted.pem"), append(intPEM, []byte(st.rootPEM)...)) + writeFile(t, filepath.Join(dir, "nginx.conf"), []byte(fmt.Sprintf(`worker_processes 1; +error_log /dev/stderr info; +events {} +http { + access_log /dev/stdout; + server { + listen 127.0.0.1:%s ssl; + server_name %s; + ssl_certificate /etc/e2e/fullchain.pem; + ssl_certificate_key /etc/e2e/leaf.key; + ssl_stapling on; + ssl_stapling_verify on; + ssl_trusted_certificate /etc/e2e/trusted.pem; + location / { return 200 "served by nginx\n"; } + } +} +`, nginxPort, nginxName))) + for _, f := range []string{"fullchain.pem", "leaf.key", "trusted.pem", "nginx.conf"} { + _ = os.Chmod(filepath.Join(dir, f), 0o644) + } + _, _ = run(t, "", "docker", "rm", "-f", nginxContainer) + mustRun(t, "docker", "run", "-d", "--name", nginxContainer, "--network", "host", + "-v", dir+":/etc/e2e:ro", "-v", filepath.Join(dir, "nginx.conf")+":/etc/nginx/nginx.conf:ro", s.env.nginx) + st.nginxDir = dir + t.Cleanup(func() { + if t.Failed() { + logs, _ := run(t, "", "docker", "logs", nginxContainer) + t.Logf("nginx logs:\n%s", lastLines(logs, 40)) + } + }) + + url := "https://" + nginxName + ":" + nginxPort + "/" + resolve := nginxName + ":" + nginxPort + ":127.0.0.1" + eventually(t, "curl --cacert root.pem through nginx", time.Minute, func() (bool, string) { + out, err := run(t, "", "curl", "-sS", "--fail", "--cacert", filepath.Join(dir, "root.pem"), "--resolve", resolve, url) + return err == nil && strings.Contains(out, "served by nginx"), out + }) + sclient := func() string { + out, _ := run(t, "", "openssl", "s_client", "-connect", "127.0.0.1:"+nginxPort, "-servername", nginxName, + "-CAfile", filepath.Join(dir, "root.pem"), "-verify_return_error", "-status") + return out + } + out := sclient() + if !strings.Contains(out, "Verify return code: 0 (ok)") { + t.Fatalf("openssl s_client -verify_return_error did not verify:\n%s", out) + } + // nginx fetches the OCSP response after the first handshake that needs + // it, so the staple shows up on a later one. + eventually(t, "a good OCSP staple from nginx", 90*time.Second, func() (bool, string) { + o := sclient() + return strings.Contains(o, "OCSP Response Status: successful") && strings.Contains(o, "Cert Status: good"), stapleState(o) + }) + if out, err := run(t, "", "curl", "-sS", "--fail", "--cert-status", "--cacert", filepath.Join(dir, "root.pem"), "--resolve", resolve, url); err != nil { + t.Fatalf("curl --cert-status with a good staple: %v\n%s", err, out) + } + note(t, "curl, s_client -verify_return_error and the stapled status all good") +} + +func stapleState(sclient string) string { + for _, l := range strings.Split(sclient, "\n") { + l = strings.TrimSpace(l) + if strings.HasPrefix(l, "OCSP response:") || strings.HasPrefix(l, "OCSP Response Status") || strings.HasPrefix(l, "Cert Status") { + return l + } + } + return "no OCSP lines" +} + +func stepRevocation(t *testing.T, s *suite, st *suiteState) { + leaf, dir := st.nginxLeaf, st.nginxDir + serial := leaf.SerialNumber.Text(16) + out := st.inter.mustCtl(t, "ca", "revoke", "--serial", serial, "--reason", "1") + if !strings.Contains(strings.ToLower(out), strings.ToLower(serial)) { + t.Fatalf("revoke output does not name %s:\n%s", serial, out) + } + + ocspURL := leaf.OCSPServer[0] + req, err := ocsp.CreateRequest(leaf, st.intCert, &ocsp.RequestOptions{Hash: crypto.SHA256}) + if err != nil { + t.Fatalf("OCSP request: %v", err) + } + for _, method := range []string{http.MethodPost, http.MethodGet} { + var resp *http.Response + if method == http.MethodPost { + resp, err = http.Post(ocspURL, "application/ocsp-request", bytes.NewReader(req)) + } else { + resp, err = http.Get(ocspURL + "/" + url.PathEscape(base64.StdEncoding.EncodeToString(req))) + } + if err != nil { + t.Fatalf("OCSP %s %s: %v", method, ocspURL, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("OCSP %s: HTTP %d", method, resp.StatusCode) + } + r, err := ocsp.ParseResponseForCert(body, leaf, st.intCert) + if err != nil { + t.Fatalf("OCSP %s: parse and verify the response: %v", method, err) + } + if r.Status != ocsp.Revoked || r.RevocationReason != ocsp.KeyCompromise { + t.Fatalf("OCSP %s: status %d reason %d, want revoked (keyCompromise)", method, r.Status, r.RevocationReason) + } + } + if out := mustRun(t, "openssl", "ocsp", "-issuer", filepath.Join(dir, "intermediate.pem"), "-cert", filepath.Join(dir, "leaf.pem"), + "-url", ocspURL, "-CAfile", filepath.Join(dir, "root.pem"), "-verify_other", filepath.Join(dir, "intermediate.pem")); !strings.Contains(out, ": revoked") { + t.Fatalf("openssl ocsp: want revoked:\n%s", out) + } + + if len(leaf.CRLDistributionPoints) == 0 { + t.Fatal("leaf carries no CRL distribution point") + } + resp, err := http.Get(leaf.CRLDistributionPoints[0]) + if err != nil { + t.Fatalf("fetch the CRL: %v", err) + } + crlDER, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + crl, err := x509.ParseRevocationList(crlDER) + if err != nil { + t.Fatalf("parse the CRL: %v", err) + } + if err := crl.CheckSignatureFrom(st.intCert); err != nil { + t.Fatalf("CRL signature: %v", err) + } + listed := false + for _, rc := range crl.RevokedCertificateEntries { + if rc.SerialNumber.Cmp(leaf.SerialNumber) == 0 { + listed = true + } + } + if !listed { + t.Fatalf("the CRL (%d entries) does not list %s", len(crl.RevokedCertificateEntries), serial) + } + + // nginx refetches the status after a reload. It staples only a good + // status: on a revoked one it logs the status and stops stapling, so the + // good staple has to disappear and the log has to say why. + mustRun(t, "docker", "exec", nginxContainer, "nginx", "-s", "reload") + eventually(t, "nginx refetches the status and finds the leaf revoked", 2*time.Minute, func() (bool, string) { + o, _ := run(t, "", "openssl", "s_client", "-connect", "127.0.0.1:"+nginxPort, "-servername", nginxName, + "-CAfile", filepath.Join(dir, "root.pem"), "-status") + logs, _ := run(t, "", "docker", "logs", nginxContainer) + revokedLogged := strings.Contains(logs, `certificate status "revoked"`) + stapledRevoked := strings.Contains(o, "Cert Status: revoked") + stillGood := strings.Contains(o, "Cert Status: good") + return (stapledRevoked || revokedLogged) && !stillGood, stapleState(o) + fmt.Sprintf("; nginx logged revoked: %t", revokedLogged) + }) + resolve := nginxName + ":" + nginxPort + ":127.0.0.1" + if out, err := run(t, "", "curl", "-sS", "--fail", "--cert-status", "--cacert", filepath.Join(dir, "root.pem"), + "--resolve", resolve, "https://"+nginxName+":"+nginxPort+"/"); err == nil { + t.Fatalf("curl --cert-status accepted a revoked staple:\n%s", out) + } + note(t, "OCSP (POST and GET) and openssl ocsp say revoked, the CRL lists it, nginx drops the good staple, curl --cert-status refuses") +} + +func (st *suiteState) intConfig(s *suite, acme, est bool) string { + sum := sha256.Sum256([]byte(st.estPass)) + return intYAML(st.adminPEM, st.rootPEM, s.env.hostIP, intProtocols{ + acme: acme, est: est, eabKeyB64: st.eabKeyB64, estPassSHA256: hex.EncodeToString(sum[:]), + }) +} + +func applyConfig(t *testing.T, v *vm, yaml string) string { + t.Helper() + path := filepath.Join(v.dir, fmt.Sprintf("apply-%d.yaml", time.Now().UnixNano())) + writeFile(t, path, []byte(yaml)) + return v.mustCtl(t, "config", "apply", "-f", path) +} + +func stepProtocolsOn(t *testing.T, s *suite, st *suiteState) { + if status := st.inter.status(t); !strings.Contains(status, "Protocols: ACME off, EST off") { + t.Fatalf("before the switch, want ACME and EST off:\n%s", status) + } + out := applyConfig(t, st.inter, st.intConfig(s, true, true)) + if !strings.Contains(out, "requires_reboot=true") { + t.Fatalf("config apply with ACME and EST on: want requires_reboot=true:\n%s", out) + } + status := st.inter.status(t) + for _, want := range []string{ + "ACME on (not running, reboot pending)", "EST on (not running, reboot pending)", "Reboot: pending", + } { + if !strings.Contains(status, want) { + t.Fatalf("after apply, status missing %q:\n%s", want, status) + } + } + st.inter.reboot(t, suiteIntCN) + status = st.inter.status(t) + if !strings.Contains(status, "Protocols: ACME on, EST on, SCEP off\n") || strings.Contains(status, "Reboot:") { + t.Fatalf("after the reboot, want ACME and EST running and no pending reboot:\n%s", status) + } + resp, err := (&http.Client{Timeout: 10 * time.Second}).Get("http://127.0.0.1:" + intACMEPort + "/acme/directory") + if err != nil { + t.Fatalf("the ACME origin does not answer after the reboot: %v", err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("ACME directory: HTTP %d after the reboot", resp.StatusCode) + } + note(t, "pending shown after apply, cleared by the reboot") +} + +func stepProtocolsOff(t *testing.T, s *suite, st *suiteState) { + out := applyConfig(t, st.inter, st.intConfig(s, false, true)) + if !strings.Contains(out, "requires_reboot=true") { + t.Fatalf("config apply with ACME off: want requires_reboot=true:\n%s", out) + } + status := st.inter.status(t) + if !strings.Contains(status, "ACME off (still running, reboot pending)") || !strings.Contains(status, "Reboot: pending") { + t.Fatalf("after apply, want ACME still running with a reboot pending:\n%s", status) + } + st.inter.reboot(t, suiteIntCN) + status = st.inter.status(t) + if !strings.Contains(status, "Protocols: ACME off, EST on, SCEP off\n") || strings.Contains(status, "Reboot:") { + t.Fatalf("after the reboot, want ACME off, EST on and nothing pending:\n%s", status) + } + if c, err := net.DialTimeout("tcp", "127.0.0.1:"+intACMEPort, 3*time.Second); err == nil { + // QEMU accepts on the host side of the forward even with nothing behind + // it, so a refused request, not a refused dial, proves it is off. + _ = c.Close() + } + resp, err := (&http.Client{Timeout: 5 * time.Second}).Get("http://127.0.0.1:" + intACMEPort + "/acme/directory") + if err == nil { + _ = resp.Body.Close() + t.Fatalf("the ACME port still answers HTTP %d after the switch off", resp.StatusCode) + } + note(t, "still running until the reboot, off after it") +} + +func stepEST(t *testing.T, s *suite, st *suiteState) { + dir := filepath.Join(s.env.out, "est") + _ = os.MkdirAll(dir, 0o755) + base := "https://127.0.0.1:" + intESTPort + "/.well-known/est" + rootFile := filepath.Join(s.env.out, "root.pem") + + enroll := func(op string, key crypto.Signer, extra ...string) *x509.Certificate { + t.Helper() + csr := filepath.Join(dir, op+".b64") + writeFile(t, csr, []byte(base64.StdEncoding.EncodeToString(newCSR(t, key, estClientName)))) + resp := filepath.Join(dir, op+".p7.b64") + args := append([]string{"-sS", "--fail-with-body", "--cacert", rootFile, + "-H", "Content-Type: application/pkcs10", "--data-binary", "@" + csr, "-o", resp}, extra...) + args = append(args, base+"/"+op) + if out, err := run(t, "", "curl", args...); err != nil { + body, _ := os.ReadFile(resp) + t.Fatalf("EST %s: %v\n%s\n%s", op, err, out, body) + } + b64, _ := os.ReadFile(resp) + der, err := base64.StdEncoding.DecodeString(strings.Join(strings.Fields(string(b64)), "")) + if err != nil { + t.Fatalf("EST %s: the response is not base64: %v", op, err) + } + p7 := filepath.Join(dir, op+".p7") + writeFile(t, p7, der) + certs := parsePEMCerts(t, []byte(mustRun(t, "openssl", "pkcs7", "-inform", "DER", "-in", p7, "-print_certs"))) + var leaf *x509.Certificate + var rest []*x509.Certificate + for _, c := range certs { + if c.IsCA { + rest = append(rest, c) + } else { + leaf = c + } + } + if leaf == nil { + t.Fatalf("EST %s: no end-entity certificate in the response", op) + } + verifyTo(t, leaf, st.rootCert, append(rest, st.intCert), estClientName, x509.ExtKeyUsageClientAuth) + return leaf + } + + cacerts, err := run(t, "", "curl", "-sS", "--fail", "--cacert", rootFile, base+"/cacerts") + if err != nil || len(strings.TrimSpace(cacerts)) == 0 { + t.Fatalf("EST cacerts: %v\n%s", err, cacerts) + } + if out, err := run(t, "", "curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "--cacert", rootFile, + "-u", estUser+":wrong-password", "-H", "Content-Type: application/pkcs10", "--data-binary", "x", base+"/simpleenroll"); err != nil || out != "401" { + t.Fatalf("EST simpleenroll with a wrong password: want HTTP 401, got %q (%v)", out, err) + } + key := p384(t) + first := enroll("simpleenroll", key, "-u", estUser+":"+st.estPass) + certFile, keyFile := filepath.Join(dir, "enrolled.pem"), filepath.Join(dir, "enrolled.key") + writeFile(t, certFile, pemBlock("CERTIFICATE", first.Raw)) + writeFile(t, keyFile, keyPEM(t, key)) + second := enroll("simplereenroll", p384(t), "--cert", certFile, "--key", keyFile) + if first.SerialNumber.Cmp(second.SerialNumber) == 0 { + t.Fatal("simplereenroll returned the same serial") + } + for _, c := range []*x509.Certificate{first, second} { + if !listsSerial(t, st.inter, c.SerialNumber, "est-client") { + t.Fatalf("ca list-issued does not show EST serial %s", c.SerialNumber.Text(16)) + } + } + note(t, fmt.Sprintf("enrolled %s, re-enrolled %s", first.SerialNumber.Text(16), second.SerialNumber.Text(16))) +} + +func stepRecertify(t *testing.T, s *suite, st *suiteState) { + before := st.intCert + csr := st.inter.mustCtl(t, "ca", "get-renewal-csr") + csrPath := filepath.Join(st.inter.dir, "renewal.csr") + writeFile(t, csrPath, []byte(csr)) + chain := st.root.mustCtl(t, "ca", "sign-subordinate", "--csr", csrPath, "--profile", "sub-ca") + chainPath := filepath.Join(st.inter.dir, "renewal-chain.pem") + writeFile(t, chainPath, []byte(chain)) + st.inter.mustCtl(t, "ca", "submit-renewed-cert", "--chain", chainPath) + after := parsePEMCerts(t, []byte(st.inter.mustCtl(t, "identity", "show", "-o", "pem")))[0] + if after.SerialNumber.Cmp(before.SerialNumber) == 0 { + t.Fatal("re-certify kept the old certificate") + } + if !bytes.Equal(after.RawSubjectPublicKeyInfo, before.RawSubjectPublicKeyInfo) { + t.Fatal("re-certify changed the Intermediate's key") + } + roots := x509.NewCertPool() + roots.AddCert(st.rootCert) + if _, err := after.Verify(x509.VerifyOptions{Roots: roots, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}}); err != nil { + t.Fatalf("re-certified Intermediate does not verify to the Root: %v", err) + } + st.intCert = after + writeFile(t, filepath.Join(s.env.out, "intermediate.pem"), pemBlock("CERTIFICATE", after.Raw)) + // A leaf issued now chains through the new certificate. + key := p384(t) + dir := filepath.Join(s.env.out, "leaves") + _ = os.MkdirAll(dir, 0o755) + leafCSR := filepath.Join(dir, "after-recertify.csr") + writeFile(t, leafCSR, pemBlock("CERTIFICATE REQUEST", newCSR(t, key, "after-recertify.cryptos.test"))) + leaf := parsePEMCerts(t, []byte(st.inter.mustCtl(t, "ca", "issue-leaf", "--csr", leafCSR, "--profile", "tls-server", "--dns", "after-recertify.cryptos.test")))[0] + verifyTo(t, leaf, st.rootCert, []*x509.Certificate{after}, "after-recertify.cryptos.test", x509.ExtKeyUsageServerAuth) + note(t, fmt.Sprintf("serial %s -> %s, same key", before.SerialNumber.Text(16), after.SerialNumber.Text(16))) +} + +// stepUpgrade stages the successor image on v, activates it and checks the +// node came back on it with its identity and state, then issues through it. +func stepUpgrade(t *testing.T, s *suite, st *suiteState, v *vm, cn string) { + if s.env.nextUKI == "" { + note(t, "no successor image (E2E_IMAGE_NEXT_UKI)") + t.Skip("no successor image") + } + before := v.mustCtl(t, "image", "status") + idBefore := v.mustCtl(t, "identity", "show", "-o", "pem") + out := v.mustCtl(t, "image", "stage", "--image", s.env.nextUKI) + t.Logf("%s: image stage:\n%s", v.name, out) + staged := v.mustCtl(t, "image", "status") + if !strings.Contains(staged, "Reboot pending: yes") { + t.Fatalf("%s: after staging, want a reboot pending:\n%s", v.name, staged) + } + bootsBefore := bootCount(v.status(t)) + started := time.Now() + v.mustCtl(t, "image", "activate", "--confirm", cn) + v.waitUp(t, bootsBefore, started, "upgrade") + after := v.mustCtl(t, "image", "status") + if !strings.Contains(after, "-next") || !strings.Contains(after, "Reboot pending: no") { + t.Fatalf("%s: after the upgrade, want the successor running and nothing pending:\nbefore:\n%s\nafter:\n%s", v.name, before, after) + } + if id := v.mustCtl(t, "identity", "show", "-o", "pem"); id != idBefore { + t.Fatalf("%s: the identity changed across the upgrade", v.name) + } + if status := v.status(t); !strings.Contains(status, "Identity: ESTABLISHED") || !strings.Contains(status, "-next") { + t.Fatalf("%s: after the upgrade:\n%s", v.name, status) + } + dir := filepath.Join(s.env.out, "leaves") + _ = os.MkdirAll(dir, 0o755) + if v == st.root { + // The Root still signs: re-certify the Intermediate through it. + csr := filepath.Join(st.inter.dir, "post-upgrade.csr") + writeFile(t, csr, []byte(st.inter.mustCtl(t, "ca", "get-renewal-csr"))) + chain := v.mustCtl(t, "ca", "sign-subordinate", "--csr", csr, "--profile", "sub-ca") + c := parsePEMCerts(t, []byte(chain))[0] + if !bytes.Equal(c.RawSubjectPublicKeyInfo, st.intCert.RawSubjectPublicKeyInfo) { + t.Fatal("the upgraded Root signed a different key") + } + roots := x509.NewCertPool() + roots.AddCert(st.rootCert) + if _, err := c.Verify(x509.VerifyOptions{Roots: roots, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}}); err != nil { + t.Fatalf("certificate from the upgraded Root does not verify: %v", err) + } + } else { + key := p384(t) + csr := filepath.Join(dir, "post-upgrade.csr") + writeFile(t, csr, pemBlock("CERTIFICATE REQUEST", newCSR(t, key, "post-upgrade.cryptos.test"))) + leaf := parsePEMCerts(t, []byte(v.mustCtl(t, "ca", "issue-leaf", "--csr", csr, "--profile", "tls-server", "--dns", "post-upgrade.cryptos.test")))[0] + verifyTo(t, leaf, st.rootCert, []*x509.Certificate{st.intCert}, "post-upgrade.cryptos.test", x509.ExtKeyUsageServerAuth) + } + note(t, "staged, activated, back on the successor with the same identity, and signing") +} + +func stepEscrowExport(t *testing.T, s *suite, st *suiteState) { + dir := filepath.Join(s.env.out, "escrow") + _ = os.MkdirAll(dir, 0o755) + out, err := st.inter.ctl("a-passphrase-long-enough\na-passphrase-long-enough\n", "ca", "export-key", "--out", filepath.Join(dir, "int.bak"), "--yes") + if err == nil || !strings.Contains(out, "non-exportable (TPM-backed)") { + t.Fatalf("export on the TPM node: want non-exportable, got err=%v:\n%s", err, out) + } + short := "seventeen-bytes!!" // 17 bytes: one under the floor + out, err = st.root.ctl(short+"\n"+short+"\n", "ca", "export-key", "--out", filepath.Join(dir, "short.bak"), "--yes") + if err == nil || !strings.Contains(out, "at least 18 bytes") { + t.Fatalf("export with a 17-byte passphrase: want refused, got err=%v:\n%s", err, out) + } + st.backupPwd = "e2e-escrow-" + hex.EncodeToString(randomBytes(8)) + st.backup = filepath.Join(dir, "root.bak") + if out, err := st.root.ctl(st.backupPwd+"\n"+st.backupPwd+"\n", "ca", "export-key", "--out", st.backup, "--yes"); err != nil { + t.Fatalf("export from the Root: %v\n%s", err, out) + } + if fi, err := os.Stat(st.backup); err != nil || fi.Size() == 0 { + t.Fatalf("no backup written: %v", err) + } + note(t, "TPM node refuses, 17 bytes refused, Root exported") +} + +func randomBytes(n int) []byte { + b := make([]byte, n) + _, _ = rand.Read(b) + return b +} + +func stepEscrowImport(t *testing.T, s *suite, st *suiteState) { + st.spare = s.newVM(t, "spare", "564d0003-0000-0000-0000-00000000e2e3", spareMgmtPort, nil, rootYAML(st.adminPEM, "e2e-spare"), st.admin) + st.spare.boot(t) + if status := st.spare.status(t); !strings.Contains(status, "Identity: NONE") { + t.Fatalf("fresh node: want no identity:\n%s", status) + } + if out, err := st.spare.ctl("wrong-passphrase-wrong-passphrase\n", "ca", "import-key", "--backup", st.backup); err == nil || !strings.Contains(out, "bad passphrase") { + t.Fatalf("import with a wrong passphrase: want refused, got err=%v:\n%s", err, out) + } + if out, err := st.spare.ctl(st.backupPwd+"\n", "ca", "import-key", "--backup", st.backup); err != nil { + t.Fatalf("import the Root backup: %v\n%s", err, out) + } + st.spare.trustCA(t) + got := parsePEMCerts(t, []byte(st.spare.mustCtl(t, "identity", "show", "-o", "pem")))[0] + if !bytes.Equal(got.Raw, st.rootCert.Raw) { + t.Fatalf("the restored node holds %s (serial %s), not the Root", got.Subject, got.SerialNumber.Text(16)) + } + if status := st.spare.status(t); !strings.Contains(status, "Identity: ESTABLISHED") { + t.Fatalf("restored node not ESTABLISHED:\n%s", status) + } + note(t, "wrong passphrase refused; the restored node holds the Root certificate") +} + +func stepConsoleReset(t *testing.T, s *suite, st *suiteState) { + v := st.spare + waitForSerial(t, v.serial, "^R reset (destroys this CA)", time.Minute) + offset := v.serial.len() + if _, err := v.keys.Write([]byte{0x12}); err != nil { + t.Fatalf("send Ctrl-R: %v", err) + } + waitForSerialSince(t, v.serial, offset, "Type the Root CA CN to confirm", 30*time.Second) + if _, err := v.keys.Write([]byte(suiteRootCN + "\r")); err != nil { + t.Fatalf("type the CN: %v", err) + } + waitForSerialSince(t, v.serial, offset, "RESET IN PROGRESS", 30*time.Second) + waitForSerialSince(t, v.serial, offset, "REPROVISION mode", 3*time.Minute) + note(t, "Ctrl-R, CN, Enter: wiped and back in re-provision maintenance") +} + +func waitForSerialSince(t *testing.T, log *serialLog, offset int, sub string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for !strings.Contains(log.since(offset), sub) { + if time.Now().After(deadline) { + t.Fatalf("serial did not show %q within %s:\n%s", sub, timeout, lastLines(log.String(), 40)) + } + time.Sleep(500 * time.Millisecond) + } +} + +// ---- ACME with cert-manager ----------------------------------------------------- + +func stepACME(t *testing.T, s *suite, st *suiteState) { + e := s.env + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + + // The node serves ACME over plain HTTP behind a TLS terminator + // (docs/acme.md). The front's certificate comes from the booted + // Intermediate, so cert-manager trusts it through the Root. + frontKey := p384(t) + dir := filepath.Join(e.out, "acme") + _ = os.MkdirAll(dir, 0o755) + csr := filepath.Join(dir, "front.csr") + writeFile(t, csr, pemBlock("CERTIFICATE REQUEST", newCSR(t, frontKey, "cryptos-acme-front"))) + front := parsePEMCerts(t, []byte(st.inter.mustCtl(t, "ca", "issue-leaf", "--csr", csr, "--profile", "acme-front")))[0] + origin := &url.URL{Scheme: "http", Host: "127.0.0.1:" + intACMEPort} + proxy := httputil.NewSingleHostReverseProxy(origin) + lis, err := net.Listen("tcp", net.JoinHostPort(e.hostIP, acmeFrontPort)) + if err != nil { + t.Fatalf("listen for the ACME TLS front: %v", err) + } + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Logf("ACME front: %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr) + proxy.ServeHTTP(w, r) + }), + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{{Certificate: [][]byte{front.Raw, st.intCert.Raw}, PrivateKey: frontKey}}, + MinVersion: tls.VersionTLS12, + }, + ReadHeaderTimeout: 5 * time.Second, + } + go func() { _ = srv.ServeTLS(lis, "", "") }() + defer func() { _ = srv.Close() }() + baseURL := "https://" + net.JoinHostPort(e.hostIP, acmeFrontPort) + "/acme" + + kc := kubectl{t: t, kubeconfig: e.kubeconfig, bin: e.kubectl} + manifest := fmt.Sprintf(`apiVersion: v1 +kind: Secret +metadata: {name: cryptos-acme-eab, namespace: cert-manager} +type: Opaque +stringData: {hmac: %[1]s} +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: {name: cryptos-image} +spec: + acme: + server: %[2]s/directory + caBundle: %[3]s + privateKeySecretRef: {name: cryptos-image-account} + externalAccountBinding: + keyID: %[4]s + keySecretRef: {name: cryptos-acme-eab, key: hmac} + solvers: + - http01: + ingress: {ingressClassName: contour} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: {name: %[5]s, namespace: %[6]s} +spec: + secretName: %[7]s + dnsNames: [%[8]s] + privateKey: {algorithm: ECDSA, size: 384} + issuerRef: {kind: ClusterIssuer, name: cryptos-image} +`, st.eabKeyB64, baseURL, base64.StdEncoding.EncodeToString([]byte(st.rootPEM)), acmeEABKeyID, + kindCert, kindNS, kindSecret, e.hostname) + deadline := time.Now().Add(90 * time.Second) + for { + if _, err := kc.run(ctx, manifest, "apply", "-f", "-"); err == nil { + break + } else if time.Now().After(deadline) { + t.Fatalf("apply the ClusterIssuer and Certificate: %v", err) + } else { + t.Logf("apply refused, retrying: %v", err) + } + time.Sleep(5 * time.Second) + } + + first := kc.waitForCertificate(ctx, "", 6*time.Minute) + s.checkACMECert(t, st, first) + firstSerial := first.leaf.SerialNumber.Text(16) + patch := fmt.Sprintf(`[{"op":"add","path":"/status/conditions/-","value":{"type":"Issuing","status":"True","reason":"ManuallyTriggered","message":"re-issuance triggered by the CryptOS full-image suite","lastTransitionTime":%q}}]`, + time.Now().UTC().Format(time.RFC3339)) + if _, err := kc.run(ctx, "", "-n", kindNS, "patch", "certificate", kindCert, "--subresource=status", "--type=json", "-p", patch); err != nil { + t.Fatalf("trigger the renewal: %v", err) + } + second := kc.waitForCertificate(ctx, firstSerial, 6*time.Minute) + s.checkACMECert(t, st, second) + note(t, fmt.Sprintf("serial %s, renewed to %s, both in list-issued", firstSerial, second.leaf.SerialNumber.Text(16))) +} + +func (s *suite) checkACMECert(t *testing.T, st *suiteState, c acmeCert) { + t.Helper() + verifyTo(t, c.leaf, st.rootCert, c.chain, s.env.hostname, x509.ExtKeyUsageServerAuth) + if len(c.leaf.DNSNames) != 1 || c.leaf.DNSNames[0] != s.env.hostname { + t.Fatalf("ACME leaf SANs %v, want [%s]", c.leaf.DNSNames, s.env.hostname) + } + if pub, ok := c.leaf.PublicKey.(*ecdsa.PublicKey); !ok || pub.Curve != elliptic.P384() { + t.Fatalf("ACME leaf key %T, want ECDSA P-384", c.leaf.PublicKey) + } + if !listsSerial(t, st.inter, c.leaf.SerialNumber, acmeLeafProfile) { + t.Fatalf("ca list-issued on the Intermediate does not show ACME serial %s", c.leaf.SerialNumber.Text(16)) + } +} + +type acmeCert struct { + leaf *x509.Certificate + chain []*x509.Certificate +} + +type kubectl struct { + t *testing.T + kubeconfig, bin string +} + +func (k kubectl) run(ctx context.Context, stdin string, args ...string) (string, error) { + cctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + cmd := exec.CommandContext(cctx, k.bin, append([]string{"--kubeconfig", k.kubeconfig}, args...)...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + return stdout.String(), fmt.Errorf("kubectl %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return stdout.String(), nil +} + +func (k kubectl) waitForCertificate(ctx context.Context, notSerial string, timeout time.Duration) acmeCert { + t := k.t + t.Helper() + deadline := time.Now().Add(timeout) + last := "" + for { + ready, _ := k.run(ctx, "", "-n", kindNS, "get", "certificate", kindCert, + "-o", `jsonpath={.status.conditions[?(@.type=="Ready")].status}/{.status.conditions[?(@.type=="Ready")].reason}`) + if ready != last { + t.Logf("Certificate %s/%s Ready=%s", kindNS, kindCert, ready) + last = ready + } + if strings.HasPrefix(ready, "True/") { + crt, err := k.run(ctx, "", "-n", kindNS, "get", "secret", kindSecret, "-o", `jsonpath={.data.tls\.crt}`) + if err == nil { + if raw, derr := base64.StdEncoding.DecodeString(strings.TrimSpace(crt)); derr == nil && len(raw) > 0 { + certs := parsePEMCerts(t, raw) + if certs[0].SerialNumber.Text(16) != notSerial { + return acmeCert{leaf: certs[0], chain: certs[1:]} + } + } + } + } + if time.Now().After(deadline) { + for _, args := range [][]string{ + {"-n", kindNS, "describe", "certificate,certificaterequest,order,challenge"}, + {"describe", "clusterissuer", "cryptos-image"}, + {"-n", "cert-manager", "logs", "deploy/cert-manager", "--tail=100"}, + } { + out, _ := k.run(context.Background(), "", args...) + t.Logf("--- kubectl %s ---\n%s", strings.Join(args, " "), out) + } + t.Fatalf("Certificate %s/%s got no new certificate within %s (last Ready=%q)", kindNS, kindCert, timeout, last) + } + time.Sleep(3 * time.Second) + } +}