test.yml

name: Test

on:
  push: {}
  workflow_dispatch:
    inputs:
      pr_head_repo:
        description: "Fork repository containing the PR commit"
        required: false
        type: string
      pr_head_sha:
        description: "Exact PR commit to test"
        required: false
        type: string

# Every branch push triggers a full run and superseded runs used to keep
# running, holding self-hosted slots that every repo on the shared pool
# queues behind. Group by ref so a newer push cancels the older run for
# that branch. main and workflow_dispatch runs get run_id, a unique group
# each, because sharing a group is unsafe even with cancellation off:
# GitHub cancels an existing pending run when a newer one enters the
# group, which would drop a commit's only test signal.
concurrency:
  group: test-${{ (github.event_name == 'push' && github.ref != 'refs/heads/main') && github.ref || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'push' && github.ref != 'refs/heads/main' }}

# A slash-command dispatch supplies a fork repository and immutable commit SHA.
# Normal push runs use the upstream repository and pushed ref.
env:
  TEST_PR_HEAD_REPO: ${{ inputs.pr_head_repo || github.repository }}
  TEST_SOURCE_REPO: ${{ inputs.pr_head_repo || github.repository }}
  TEST_SOURCE_REF: ${{ inputs.pr_head_sha || github.ref }}

permissions:
  contents: read

jobs:
  test:
    runs-on: [self-hosted, linux, x64, kvm]
    steps:
      - uses: actions/checkout@v4
        with:
          clean: false
          repository: ${{ env.TEST_SOURCE_REPO }}
          ref: ${{ env.TEST_SOURCE_REF }}
          persist-credentials: false

      - name: Clean workspace (preserve binary caches)
        run: |
          git reset --hard HEAD
          git clean -ffdx \
            -e lib/vmm/binaries/ \
            -e lib/hypervisor/firecracker/binaries/ \
            -e lib/ingress/binaries/ \
            -e bin/
      
      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          # Not necessary to upload cache on self-hosted runner(s)
          # ~/go/pkg/mod and ~/.cache/go-build stay on disk between runs automatically.
          cache: false
          go-version: '1.25.4'

      - name: Check UFFD pager version
        run: |
          git fetch "https://github.com/${{ github.repository }}.git" main --depth=1
          bash scripts/check-uffd-version.sh FETCH_HEAD
      
      - name: Install dependencies
        run: |
          set -xe
          apt_update_with_retry() {
            local attempts=5
            local sleep_seconds=30
            local n=1
            while [ "$n" -le "$attempts" ]; do
              if timeout 120s sudo apt-get update; then
                return 0
              fi
              if [ "$n" -eq "$attempts" ]; then
                return 1
              fi
              echo "apt-get update failed (attempt ${n}/${attempts}); retrying in ${sleep_seconds}s..."
              sleep "$sleep_seconds"
              n=$((n + 1))
            done
          }
          if ! command -v mkfs.erofs &> /dev/null || \
             ! command -v mkfs.ext4 &> /dev/null || \
             ! command -v iptables &> /dev/null || \
             ! command -v qemu-system-x86_64 &> /dev/null || \
             ! qemu-system-x86_64 --version >/dev/null 2>&1; then
            apt_update_with_retry
            timeout 300s sudo apt-get install -y erofs-utils e2fsprogs iptables qemu-system-x86 qemu-utils
          fi
          go mod download

      - name: Verify Linux test toolchain
        run: |
          set -euo pipefail
          TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH"
          for bin in mkfs.erofs mkfs.ext4 iptables qemu-system-x86_64; do
            if ! sudo env "PATH=$TEST_PATH" bash -lc "command -v '$bin' >/dev/null"; then
              echo "missing required binary under sudo PATH: $bin"
              exit 1
            fi
            sudo env "PATH=$TEST_PATH" bash -lc "command -v '$bin'"
          done

      # Slash-command runs are maintainer-approved and need authenticated pulls
      # for images that are not covered by the prewarm cache.
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      - name: Generate OpenAPI code
        run: make oapi-generate

      - name: Download Cloud Hypervisor binaries
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          for version in v49.0 v51.1; do
            for architecture in x86_64 aarch64; do
              if [ "$architecture" = "x86_64" ]; then
                asset="cloud-hypervisor-static"
                file_pattern='x86-64'
              else
                asset="cloud-hypervisor-static-aarch64"
                file_pattern='aarch64|ARM aarch64'
              fi
              destination="lib/vmm/binaries/cloud-hypervisor/$version/$architecture/cloud-hypervisor"
              mkdir -p "$(dirname "$destination")"
              gh release download "$version" \
                --repo cloud-hypervisor/cloud-hypervisor \
                --pattern "$asset" \
                --output "$destination" \
                --clobber
              chmod +x "$destination"
              file "$destination" | grep -Eiq "$file_pattern"
            done
          done

      - name: Build
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          for attempt in 1 2 3; do
            if make build; then
              exit 0
            fi
            if [ "$attempt" -lt 3 ]; then
              echo "build failed (attempt ${attempt}/3); retrying in 10 seconds..."
              sleep 10
            fi
          done
          exit 1

      - name: Install UFFD pager systemd template
        run: |
          sudo mkdir -p /run/hypeman/uffd
          sudo tee /etc/systemd/system/hypeman-uffd@.service > /dev/null <<'EOF'
          [Unit]
          Description=Hypeman UFFD Pager (%i)
          Documentation=https://github.com/kernel/hypeman
          After=network.target

          [Service]
          Type=simple
          Environment="HYPEMAN_UFFD_BINARY=/opt/hypeman/bin/hypeman-uffd-pager"
          Environment="HYPEMAN_UFFD_DATA_DIR=/var/lib/hypeman"
          Environment="HYPEMAN_UFFD_VERSION_KEY=%i"
          Environment="HYPEMAN_UFFD_CACHE_MAX_BYTES=4294967296"
          EnvironmentFile=-/run/hypeman/uffd/%i.env
          ExecStart=/bin/sh -c 'exec "$HYPEMAN_UFFD_BINARY" --data-dir "$HYPEMAN_UFFD_DATA_DIR" --version-key "$HYPEMAN_UFFD_VERSION_KEY" --cache-max-bytes "$HYPEMAN_UFFD_CACHE_MAX_BYTES"'
          Restart=on-failure
          RestartSec=5
          KillMode=process
          EOF
          sudo systemctl daemon-reload

      - name: Prewarm test cache
        env:
          DOCKER_CONFIG: /home/debianuser/.docker
          GITHUB_TOKEN: ${{ github.token }}
          HYPEMAN_TEST_REGISTRY: 127.0.0.1:5001
        run: |
          # Prewarm cache must live on the same filesystem as the test TMPDIR
          # (/ci): the image build symlinks the oci-cache from here into each
          # per-test cache and builds rootfs there, and those operations are not
          # cross-filesystem safe. Keeping both on /ci avoids that.
          export HYPEMAN_TEST_PREWARM_DIR="/ci/prewarm/linux-amd64"
          mkdir -p "$HYPEMAN_TEST_PREWARM_DIR"
          sudo chown -R "$(id -u):$(id -g)" "$HYPEMAN_TEST_PREWARM_DIR"
          for attempt in 1 2 3; do
            if go run ./cmd/test-prewarm; then
              exit 0
            fi
            if [ "$attempt" -lt 3 ]; then
              echo "prewarm failed (attempt ${attempt}/3); retrying in 10 seconds..."
              sleep 10
            fi
          done
          exit 1

      - name: Check gofmt
        run: |
          set -euo pipefail
          go_files="$(git ls-files '*.go')"
          if [ -z "$go_files" ]; then
            exit 0
          fi
          unformatted="$(echo "$go_files" | xargs gofmt -l)"
          if [ -n "$unformatted" ]; then
            echo "The following files are not gofmt-formatted:"
            echo "$unformatted"
            exit 1
          fi
     
      - name: Run tests
        env:
          GO_TEST_TIMEOUT: 600s
          GITHUB_TOKEN: ${{ github.token }}
          # Docker auth for tests running as root (sudo)
          DOCKER_CONFIG: /home/debianuser/.docker
          # TLS/ACME testing (optional - tests will skip if not configured)
          ACME_EMAIL: ${{ env.TEST_PR_HEAD_REPO == github.repository && secrets.ACME_EMAIL || '' }}
          ACME_DNS_PROVIDER: "cloudflare"
          ACME_CA: "https://acme-staging-v02.api.letsencrypt.org/directory"
          CLOUDFLARE_API_TOKEN: ${{ env.TEST_PR_HEAD_REPO == github.repository && secrets.CLOUDFLARE_API_TOKEN || '' }}
          TLS_TEST_DOMAIN: "test.hypeman-development.com"
          TLS_ALLOWED_DOMAINS: '*.hypeman-development.com'
          HYPEMAN_TEST_PREWARM_STRICT: "1"
          HYPEMAN_TEST_REFLINK_STRICT: "1"
          HYPEMAN_TEST_REGISTRY: 127.0.0.1:5001
          HYPEMAN_UFFD_PAGER_BINARY: ${{ runner.temp }}/hypeman-uffd-pager-${{ github.run_id }}-${{ github.run_attempt }}
          HYPEMAN_UFFD_SYSTEMD_INSTANCE_PREFIX: ci-${{ github.run_id }}-${{ github.run_attempt }}
          # Fast scratch filesystem provisioned on the runner. Must be reflink-capable
          # (the VM disk fork fast path uses FICLONE) and kept short to stay under the
          # 108-char AF_UNIX sun_path limit for VMM control sockets.
          TMPDIR: /ci
        run: |
          cp "$PWD/bin/hypeman-uffd-pager" "$HYPEMAN_UFFD_PAGER_BINARY"
          chmod +x "$HYPEMAN_UFFD_PAGER_BINARY"
          export HYPEMAN_TEST_PREWARM_DIR="/ci/prewarm/linux-amd64"
          make test TEST_TIMEOUT=20m

      - name: Cleanup
        if: always()
        run: |
          set -euo pipefail

          run_prefix="ci-${{ github.run_id }}-${{ github.run_attempt }}"
          guest_pids=()
          mapfile -t vmm_pids < <(pgrep -x 'cloud-hyperviso|firecracker|qemu-system-.*' || true)
          for pid in "${vmm_pids[@]}"; do
            proc="/proc/$pid"
            cmdline="$(cat "$proc/cmdline" 2>/dev/null | tr '\0' ' ' || true)"
            if [[ "$cmdline" == *"/ci/"* ]] && sudo grep -zFqx "HYPEMAN_UFFD_SYSTEMD_INSTANCE_PREFIX=$run_prefix" "$proc/environ" 2>/dev/null; then
              guest_pids+=("$pid")
            fi
          done

          if (( ${#guest_pids[@]} > 0 )); then
            echo "Stopping leftover test VMs: ${guest_pids[*]}"
            sudo kill -TERM "${guest_pids[@]}" 2>/dev/null || true
            sleep 2
            for pid in "${guest_pids[@]}"; do
              if sudo kill -0 "$pid" 2>/dev/null; then
                sudo kill -KILL "$pid" 2>/dev/null || true
              fi
            done
          fi

          units="$(systemctl list-units --all --full --plain 'hypeman-uffd@ci-${{ github.run_id }}-${{ github.run_attempt }}-*.service' --no-legend | awk '{print $1}' || true)"
          if [ -n "$units" ]; then
            echo "$units" | xargs -r sudo systemctl stop || true
            echo "$units" | xargs -r sudo systemctl reset-failed || true
          fi
          sudo rm -f /run/hypeman/uffd/ci-${{ github.run_id }}-${{ github.run_attempt }}-*.env
          rm -f "${{ runner.temp }}/hypeman-uffd-pager-${{ github.run_id }}-${{ github.run_attempt }}"

  test-darwin:
    runs-on: [self-hosted, macos, arm64]
    concurrency:
      group: macos-ci-test-${{ inputs.pr_head_sha || github.ref }}
      cancel-in-progress: true
    env:
      DATA_DIR: /tmp/hypeman-ci-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v4
        with:
          repository: ${{ env.TEST_SOURCE_REPO }}
          ref: ${{ env.TEST_SOURCE_REF }}
          persist-credentials: false
      - uses: actions/setup-go@v6
        with:
          go-version: '1.25.4'
          cache: false
      - name: Ensure Docker is running
        run: |
          if docker info >/dev/null 2>&1; then
            exit 0
          fi

          if colima status >/dev/null 2>&1; then
            colima stop --force || true
          fi

          if ! colima start; then
            colima stop --force || true
            if ! colima start; then
              colima delete --force
              colima start
            fi
          fi

          for attempt in {1..30}; do
            if docker info >/dev/null 2>&1; then
              exit 0
            fi
            echo "waiting for Docker daemon (${attempt}/30)"
            sleep 2
          done

          docker info
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      - name: Install dependencies
        run: |
          brew list e2fsprogs &>/dev/null || brew install e2fsprogs
          brew list erofs-utils &>/dev/null || brew install erofs-utils
          go mod download
      - name: Create run-scoped data directory
        run: mkdir -p "$DATA_DIR"
      - name: Generate OpenAPI code
        run: make oapi-generate
      - name: Build
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: make build

      - name: Prewarm test cache
        env:
          GITHUB_TOKEN: ${{ github.token }}
          HYPEMAN_TEST_REGISTRY: 127.0.0.1:5001
        run: |
          export HYPEMAN_TEST_PREWARM_DIR="$HOME/.cache/hypeman-ci/darwin-arm64"
          go run ./cmd/test-prewarm

      - name: Check gofmt
        run: |
          set -euo pipefail
          go_files="$(git ls-files '*.go')"
          if [ -z "$go_files" ]; then
            exit 0
          fi
          unformatted="$(echo "$go_files" | xargs gofmt -l)"
          if [ -n "$unformatted" ]; then
            echo "The following files are not gofmt-formatted:"
            echo "$unformatted"
            exit 1
          fi

      - name: Run tests
        env:
          GO_TEST_TIMEOUT: 600s
          DEFAULT_HYPERVISOR: vz
          GITHUB_TOKEN: ${{ github.token }}
          JWT_SECRET: ci-test-secret
          HYPEMAN_TEST_PREWARM_STRICT: "1"
          HYPEMAN_TEST_REGISTRY: 127.0.0.1:5001
        run: |
          export HYPEMAN_TEST_PREWARM_DIR="$HOME/.cache/hypeman-ci/darwin-arm64"
          make test
      - name: Run VZ builder integration test
        env:
          HYPEMAN_RUN_BUILDER_INTEGRATION_TEST: "1"
        run: |
          # Self-hosted runners retain Docker layers across jobs. Reclaim them so
          # the builder image and two VZ builds have predictable disk headroom.
          docker system prune --all --force --volumes
          PATH="/opt/homebrew/opt/e2fsprogs/sbin:$PATH" \
            go test -count=1 -tags containers_image_openpgp \
              -run='^TestBuilderPersistentCacheReuse$' -timeout=20m -v ./integration
      - name: Cleanup
        if: always()
        run: |
          pkill -f "vz-shim.*$DATA_DIR" || true
          rm -rf "$DATA_DIR"
          make clean

  e2e-install:
    runs-on: [self-hosted, macos, arm64]
    needs: test-darwin
    concurrency:
      group: macos-ci-e2e-${{ inputs.pr_head_sha || github.ref }}
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v4
        with:
          repository: ${{ env.TEST_SOURCE_REPO }}
          ref: ${{ env.TEST_SOURCE_REF }}
          persist-credentials: false
      - uses: actions/setup-go@v6
        with:
          go-version: '1.25.4'
          cache: false
      - name: Install dependencies
        run: |
          brew list caddy &>/dev/null || brew install caddy
          if ! docker info >/dev/null 2>&1; then
            colima start || {
              colima stop --force || true
              colima start
            }
          fi
          for attempt in {1..30}; do
            if docker info >/dev/null 2>&1; then
              break
            fi
            echo "waiting for Docker daemon (${attempt}/30)"
            sleep 2
          done
          docker info >/dev/null
          # Self-hosted runners retain Docker layers across jobs. Start the
          # install E2E with deterministic headroom for its builder image.
          docker system prune --all --force --volumes
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      - name: Build E2E install artifacts
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          make sign-darwin
          cp bin/hypeman bin/hypeman-api
          go build -o bin/hypeman-token ./cmd/gen-jwt
          cp config.example.darwin.yaml bin/
          docker build -t hypeman/builder:latest \
            -f lib/builds/images/generic/Dockerfile .
      - name: Run E2E install test
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          curl -fsSL --retry 5 --retry-all-errors \
            "https://raw.githubusercontent.com/${{ github.repository }}/${{ github.sha }}/scripts/e2e-install-test.sh" \
            -o "$RUNNER_TEMP/e2e-install-test.sh"
          HYPEMAN_E2E_REPO_DIR="$PWD" BINARY_DIR="$PWD/bin" \
            bash "$RUNNER_TEMP/e2e-install-test.sh"
      - name: Run E2E CLI-only install test
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          CLI_DIR="$(mktemp -d)"
          INSTALL_DIR="$CLI_DIR" bash scripts/install-cli.sh
          "$CLI_DIR/hypeman" --version
      - name: Cleanup on failure
        if: failure()
        run: bash scripts/uninstall.sh || true