test(e2e): boot the real image in QEMU as a Root and an Intermediate and run the product suite

.github/workflows/ci-e2e-image.yml

@@ -0,0 +1,199 @@
+name: Full-image suite (QEMU Root and Intermediate)
+
+# Boots the real CryptOS image in QEMU (swtpm, OVMF) as a Root and an
+# Intermediate on a GitHub-hosted runner and runs the suite of named steps in
+# test/integration/image_suite_test.go against them over the mTLS API:
+# ceremony, sign-subordinate, leaves, nginx with OCSP stapling, revocation,
+# the protocol switch, ACME with cert-manager in kind, EST, re-certify, escrow
+# and the console reset. Each step's result and the coverage the run reached
+# go into the job summary. See test/image/run.sh.
+#
+# Slow, so not on every pull request: nightly, on pull requests that touch
+# the image build, the boot or the protocol and config code, and on manual
+# dispatch. It is not a required check; the fast in-process kind test
+# (ci-kind-acme.yml) stays the pull-request check for ACME.
+#
+# The image job builds a coverage-instrumented variant of the image
+# (test/image/build.sh) and caches it by the hash of everything that goes into
+# it, so a run whose inputs did not change reuses the last build instead of
+# rebuilding. The kernel build shares ci-image.yml's cache. No release or CI
+# image is ever built with -cover.
+on:
+  schedule:
+    - cron: "37 6 * * *"
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - "build/**"
+      - "cmd/init/**"
+      - "cmd/cryptos-console/**"
+      - "cmd/cryptos-switchroot/**"
+      - "internal/init/**"
+      - "internal/acme/**"
+      - "internal/est/**"
+      - "internal/config/**"
+      - "internal/revocation/**"
+      - "test/image/**"
+      - "test/integration/**"
+      - ".github/workflows/ci-e2e-image.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  image:
+    name: Build the coverage-instrumented test image
+    if: github.event.pull_request.draft != true
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    outputs:
+      key: ${{ steps.key.outputs.key }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # buildinfo.sh describes the build from the tags.
+          fetch-depth: 0
+
+      - name: Image cache key
+        id: key
+        run: echo "key=e2e-image-uki-${{ hashFiles('build/**', 'cmd/**', 'internal/**', 'go.mod', 'go.sum', 'test/image/build.sh') }}" >>"$GITHUB_OUTPUT"
+
+      - name: Reuse a cached test image
+        id: cached
+        uses: actions/cache/restore@v4
+        with:
+          path: build/out/cryptos-amd64-e2e*
+          key: ${{ steps.key.outputs.key }}
+
+      - name: Set up Go
+        if: steps.cached.outputs.cache-hit != 'true'
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Install the image toolchain
+        if: steps.cached.outputs.cache-hit != 'true'
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          # The same list as ci-image.yml (see the notes there).
+          sudo apt-get install -y --no-install-recommends \
+            build-essential bc flex bison libelf-dev libssl-dev xz-utils \
+            squashfs-tools systemd-ukify systemd-boot-efi cpio python3-pefile
+
+      - name: Cache the kernel build
+        if: steps.cached.outputs.cache-hit != 'true'
+        uses: actions/cache@v4
+        with:
+          path: build/.work/kernel
+          key: kernel-amd64-${{ hashFiles('build/ci/versions.env', 'build/kernel/cryptos.config') }}
+
+      - name: Cache the static tools
+        if: steps.cached.outputs.cache-hit != 'true'
+        uses: actions/cache@v4
+        with:
+          path: |
+            build/out/cryptsetup-amd64
+            build/out/mke2fs-amd64
+            build/out/sgdisk-amd64
+            build/out/mkfs.vfat-amd64
+          key: e2e-static-tools-amd64-${{ hashFiles('build/ci/versions.env', 'build/cryptsetup/**', 'build/e2fsprogs/**', 'build/gptfdisk/**', 'build/dosfstools/**') }}
+
+      - name: Build the test image
+        if: steps.cached.outputs.cache-hit != 'true'
+        run: bash test/image/build.sh
+
+      - name: Save the test image
+        if: steps.cached.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: build/out/cryptos-amd64-e2e*
+          key: ${{ steps.key.outputs.key }}
+
+      - name: Hand the image to the suite
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-image-uki
+          path: build/out/cryptos-amd64-e2e*
+          retention-days: 3
+          if-no-files-found: error
+
+  suite:
+    name: Boot the image and run the suite
+    needs: image
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Fetch the test image
+        uses: actions/download-artifact@v4
+        with:
+          name: e2e-image-uki
+          path: build/out
+
+      # Hosted Linux runners expose /dev/kvm, but only to root until a udev
+      # rule opens it up (the GitHub-documented way to use it).
+      - name: Enable KVM
+        run: |
+          set -euo pipefail
+          if [ -e /dev/kvm ]; then
+            echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' |
+              sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null
+            sudo udevadm control --reload-rules
+            sudo udevadm trigger --name-match=kvm
+            ls -l /dev/kvm
+            echo "KVM: available" >>"$GITHUB_STEP_SUMMARY"
+          else
+            echo "KVM: not available on this runner; the VMs run under TCG" >>"$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Install QEMU, swtpm, OVMF and mtools
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            qemu-system-x86 qemu-utils ovmf swtpm swtpm-tools mtools gdisk
+          qemu-system-x86_64 --version | head -1
+          swtpm --version
+
+      - name: Cache kind, kubectl and manifests
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/cryptos-e2e-kind
+          key: kind-acme-${{ runner.arch }}-${{ hashFiles('test/kind/versions.env') }}
+
+      - name: Run the suite
+        env:
+          E2E_OUT: ${{ runner.temp }}/e2e-image
+        run: bash test/image/run.sh
+
+      - name: Keep the logs, serial consoles and coverage report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-image-results
+          path: |
+            ${{ runner.temp }}/e2e-image/*.md
+            ${{ runner.temp }}/e2e-image/*.log
+            ${{ runner.temp }}/e2e-image/coverage*
+            ${{ runner.temp }}/e2e-image/vm-*/serial.log
+          retention-days: 14
+          if-no-files-found: warn

README.md

@@ -45,6 +45,8 @@ internal/
   bootstrap/        # bootstrap admin cert loading + first-ceremony rotation
 build/              # kernel config, UKI assembly recipes, SquashFS templates
 test/kind/          # kind + cert-manager ACME end-to-end harness (task e2e:kind)
+test/image/         # full-image suite: build, run and coverage scripts (task e2e:image)
+test/integration/   # QEMU + swtpm tests against the booted image, including the full-image suite
 testdata/configs/   # sample machine configs
 ```
 
@@ -57,10 +59,13 @@ task ci          # fmt + lint + vet + test + build (both binaries)
 task build       # produces bin/init and bin/cryptosctl, stamped with the build identity
 task license     # re-inject Apache 2.0 headers via golic
 task e2e:kind    # Linux + docker: cert-manager in kind gets a certificate over ACME
+task e2e:image   # Linux + KVM + docker: boot the image as a Root and an Intermediate and run the full suite
 ```
 
 `task e2e:kind` builds a Root and an ACME-serving Intermediate in-process (software keys, no TPM), stands up a kind cluster with cert-manager and Contour, and checks that a `Certificate` goes Ready with a chain to the root and renews to a new serial. It downloads pinned, checksum-checked kind, kubectl and manifests ([`test/kind/versions.env`](test/kind/versions.env)), needs sudo once to add a `/etc/hosts` line for the test name, and skips when docker isn't available.
 
+`task e2e:image` builds a coverage-instrumented variant of the image ([`test/image/build.sh`](test/image/build.sh): the same kernel and rootfs, with `/init` built with `-cover` and a throwaway upgrade anchor, plus a signed successor image for the upgrade step; release and CI images are never built with `-cover`), boots it in QEMU with swtpm and OVMF as a software-key Root and a TPM-key Intermediate, and runs the steps in [`test/integration/image_suite_test.go`](test/integration/image_suite_test.go) against them with `cryptosctl`: the ceremony, sign-subordinate, leaf issuance, nginx with OCSP stapling, revocation, the ACME and EST protocol switch, cert-manager over ACME in kind, EST enrolment, re-certify, escrow, an in-place image upgrade on both nodes and the console reset. Each step reports pass, fail or skip in `summary.md`, and the merged coverage lands in `coverage.html`. It needs qemu, swtpm, OVMF, mtools, sgdisk, docker and sudo for one `/etc/hosts` line, and skips on a host without them.
+
 `bin/cryptosctl version` prints the version, commit, and build date the binary was built from (`git describe --tags --always --dirty`; see [`build/README.md`](build/README.md#build-identity)); pass `--endpoint` to also show the node's version.
 
 The image pipeline (`task image` — hardened kernel build, SquashFS rootfs, UKI assembly + Secure Boot signing) has draft recipes under `build/` that run on a Linux build host; see [`build/README.md`](build/README.md). They are written but not yet executed end to end. The QEMU + `swtpm` integration harness lands in a subsequent PR.
@@ -102,7 +107,7 @@ GitHub Actions:
 
 - **`ci-kind-acme`** ([`ci-kind-acme.yml`](.github/workflows/ci-kind-acme.yml)) — `test/kind/run.sh` (the `task e2e:kind` harness) on pull requests that touch the ACME, config, node or e2e code, on a GitHub-hosted runner. Drafts are skipped, and it isn't a required check.
 
-The QEMU + `swtpm` integration boot is run on a real host by the operator, not in CI.
+- **`ci-e2e-image`** ([`ci-e2e-image.yml`](.github/workflows/ci-e2e-image.yml)) — the full-image suite (`task e2e:image`) on a GitHub-hosted runner, with KVM when the runner has it: nightly, on pull requests that touch the image build, the boot, or the protocol and config code, and on manual dispatch. The step table and per-package coverage go into the job summary and the HTML report into the run's artifacts. Drafts are skipped, and it isn't a required check.
 
 ## 🔑 Management surfaces
 

Taskfile.yml

@@ -184,6 +184,12 @@ tasks:
     cmds:
       - bash test/kind/run.sh
 
+  e2e:image:
+    desc: "[Linux] Full-image suite: boot the image in QEMU as a Root and an Intermediate and drive every step with cryptosctl (needs KVM, qemu, swtpm, OVMF, docker; skips without them)"
+    cmds:
+      - bash test/image/build.sh
+      - bash test/image/run.sh
+
   test:integration:
     desc: "[Linux] Phase 1 QEMU + swtpm end-to-end acceptance (needs the toolchain; see test/integration)"
     cmds: