From 37072ee81cd7b294c714bfa5ecc829b6927b3d70 Mon Sep 17 00:00:00 2001 From: alangou Date: Thu, 27 Aug 2026 14:53:33 +0000 Subject: [PATCH 1/4] feat(build): publish OCI SBOM and provenance attestations (#2836) * feat(build): embed auditable Rust dependency metadata Signed-off-by: Adrien Langou * feat(build): publish OCI SBOM and provenance attestations Signed-off-by: Adrien Langou --------- Signed-off-by: Adrien Langou --- .agents/skills/sbom/SKILL.md | 17 +++++++- .github/workflows/branch-e2e.yml | 2 + .github/workflows/docker-build.yml | 15 +++++++ .github/workflows/rust-native-build.yml | 2 + architecture/build.md | 36 +++++++++++------ docs/security/verifying-images.mdx | 36 +++++++++++++++++ tasks/scripts/docker-build-image.sh | 15 ++++--- tasks/scripts/verify-image-sbom.sh | 53 +++++++++++++++++++++++++ 8 files changed, 158 insertions(+), 18 deletions(-) create mode 100644 docs/security/verifying-images.mdx create mode 100755 tasks/scripts/verify-image-sbom.sh diff --git a/.agents/skills/sbom/SKILL.md b/.agents/skills/sbom/SKILL.md index 852259293d..6996ffcf66 100644 --- a/.agents/skills/sbom/SKILL.md +++ b/.agents/skills/sbom/SKILL.md @@ -13,13 +13,28 @@ The OpenShell SBOM tooling produces source-tree CycloneDX JSON SBOMs using Syft, SBOMs are **release artifacts only** -- they are generated on demand and not committed to the repository. Output lands in `deploy/sbom/output/` (gitignored). -Release Dev and Release Tag image builds separately embed cargo-auditable metadata in the staged gateway and supervisor binaries. This metadata describes the binary's Rust dependency graph and lets Syft discover Cargo packages from the binary itself. It is not a complete image SBOM and is not an OCI SBOM attestation; publishing such an attestation remains separate work. +Pushed gateway and supervisor images carry an SPDX SBOM and minimal SLSA provenance as OCI attestations. Branch E2E, Release Dev, and Release Tag image binaries embed cargo-auditable metadata, so their image SBOMs include linked Rust crates. ## Prerequisites - `mise install` has been run (installs Syft and other tools) - The repository is checked out at the root +## Inspecting an Image SBOM + +BuildKit uses its default Syft scanner and attaches one SPDX document per platform. Read one without pulling the image: + +```bash +docker buildx imagetools inspect ghcr.io/nvidia/openshell/gateway:latest \ + --format '{{ json (index .SBOM "linux/amd64").SPDX }}' +``` + +Validate the final attestation, requiring a Cargo package for an auditable image: + +```bash +tasks/scripts/verify-image-sbom.sh ghcr.io/nvidia/openshell/gateway:latest --require-cargo +``` + ## Inspecting an Auditable Image Binary Opt into auditable metadata when staging a local image binary: diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 2b4d9d5d46..659027a8d6 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -86,6 +86,7 @@ jobs: with: component: gateway image-tag: ${{ github.sha }} + auditable: true build-supervisor: needs: [pr_metadata] @@ -97,6 +98,7 @@ jobs: with: component: supervisor image-tag: ${{ github.sha }} + auditable: true build-cli: needs: [pr_metadata] diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 2e7abe2008..73cb678e78 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -294,6 +294,10 @@ jobs: volumes: - /var/run/docker.sock:/var/run/docker.sock steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs['checkout-ref'] || github.sha }} + - name: Log in to GHCR run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin @@ -309,3 +313,14 @@ jobs: --prefer-index=false \ -t "${image}:${{ needs.resolve.outputs.image_tag_base }}" \ "${refs[@]}" + + - name: Verify merged manifest SBOM attestation + env: + IMAGE_REF: ghcr.io/nvidia/openshell/${{ inputs.component }}:${{ needs.resolve.outputs.image_tag_base }} + REQUIRE_CARGO: ${{ inputs.auditable }} + run: | + args=("${IMAGE_REF}") + if [[ "${REQUIRE_CARGO}" == "true" ]]; then + args+=(--require-cargo) + fi + tasks/scripts/verify-image-sbom.sh "${args[@]}" diff --git a/.github/workflows/rust-native-build.yml b/.github/workflows/rust-native-build.yml index 3496f4779f..24124d0ae4 100644 --- a/.github/workflows/rust-native-build.yml +++ b/.github/workflows/rust-native-build.yml @@ -40,6 +40,8 @@ on: required: false type: string default: "" + # Branch E2E and release gateway/supervisor image builds opt in. + # CLI images and standalone glibc-static supervisor validation keep false. auditable: description: "Embed cargo-auditable dependency metadata in the binary" required: false diff --git a/architecture/build.md b/architecture/build.md index a1c6974989..ff8dc424cc 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -131,27 +131,39 @@ step via the `rust-native-build.yml` workflow (per-architecture, per-component) and uploads the result as an artifact that the image build job downloads back into the staging directory before running Buildx. -Gateway and supervisor binaries staged into Release Dev and Release Tag images -are compiled through `cargo auditable` (pinned in `mise.toml`), which embeds a -`.dep-v0` section describing the Rust dependencies actually compiled into the -binary. That section holds data rather than symbols, so it survives the +Gateway and supervisor binaries staged into branch E2E, Release Dev, and Release +Tag images are compiled through `cargo auditable` (pinned in `mise.toml`), which +embeds a `.dep-v0` section describing the Rust dependencies actually compiled +into the binary. That section holds data rather than symbols, so it survives the workspace's `strip = true` release profile, and Syft can catalog the crates present in image binaries instead of inferring them from the source tree. This is a different artifact from the source SBOM produced by `syft dir:.` in -`tasks/sbom.toml`, which describes the checkout, and from an OCI SBOM -attestation, which remains out of scope. - -Only release image builds are auditable. `docker-build.yml` and -`rust-native-build.yml` take an `auditable` input that defaults to false, so PR -and E2E image builds and standalone release artifacts remain non-auditable. The -CI image gains the pinned `cargo-auditable` tool through `mise install --locked` -but ships no auditable OpenShell binary of its own. Local staging opts in with +`tasks/sbom.toml`, which describes the checkout, and from the image SBOM +attestation below, which describes a published image. + +`docker-build.yml` and `rust-native-build.yml` take an `auditable` input that +defaults to false. The branch E2E gateway and supervisor image builds set it to +true so E2E exercises release-parity image binaries; other PR image builds and +standalone release artifacts stay non-auditable. The CI image gains the pinned +`cargo-auditable` tool through `mise install --locked` but ships no auditable +OpenShell binary of its own. Local staging opts in with `OPENSHELL_AUDITABLE=1`. sccache's `RUSTC_WRAPPER` is unset only around auditable builds, because it would otherwise wrap `cargo-auditable`'s workspace wrapper and be misidentified as `rustc`. Auditable builds are verified by scanning the built binary with Syft and requiring at least one decoded Cargo package; the check runs only for those builds. +Pushed Docker images carry minimal SLSA provenance and a per-platform SPDX SBOM +generated by BuildKit's default Syft scanner. The registry exporter uses OCI +media types and `oci-artifact=true`, so each attestation identifies its subject. +GHCR exposes these through the image index because it has no referrers API. + +Attestations require a registry-backed image index. Local builds therefore keep +`--provenance=false`, and Podman builds carry neither attestation. +`tasks/scripts/verify-image-sbom.sh` verifies the merged multi-arch tag and runs +with `--require-cargo` for auditable builds, so those attestations must also +contain Cargo packages. + Runtime layout: - **Gateway**: `gcr.io/distroless/cc-debian13:nonroot` base, GNU-linked binary at diff --git a/docs/security/verifying-images.mdx b/docs/security/verifying-images.mdx new file mode 100644 index 0000000000..4b9704d3b5 --- /dev/null +++ b/docs/security/verifying-images.mdx @@ -0,0 +1,36 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Verify the Contents of Published OpenShell Images" +sidebar-title: "Verify Image Contents" +slug: "security/verify-image-contents" +description: "Read the SBOM attestation attached to published gateway and supervisor images to audit what each image contains." +keywords: "Generative AI, Cybersecurity, Supply Chain, SBOM, Container Images" +position: 2 +--- + +Published gateway and supervisor images carry one SPDX SBOM per platform as OCI attestations. + +## Inspect an Image + +Read a platform's document without pulling the image: + +```shell +docker buildx imagetools inspect ghcr.io/nvidia/openshell/gateway:latest --format '{{ json (index .SBOM "linux/amd64").SPDX }}' +``` + +List the packages instead of the full document: + +```shell +docker buildx imagetools inspect ghcr.io/nvidia/openshell/gateway:latest --format '{{ range (index .SBOM "linux/amd64").SPDX.packages }}{{ .name }}@{{ .versionInfo }}{{ println }}{{ end }}' +``` + +The same commands work for `ghcr.io/nvidia/openshell/supervisor`. + +## Coverage + +Every SBOM lists the base-image packages. Release Dev and Release Tag images also list the Rust crates compiled into their OpenShell binary. + + +OpenShell also publishes minimal SLSA provenance. It records how BuildKit produced the image, including its source revision, build platform, and base-image materials, without the extra build parameters included by full provenance. + diff --git a/tasks/scripts/docker-build-image.sh b/tasks/scripts/docker-build-image.sh index 8570180f12..08ba00e066 100755 --- a/tasks/scripts/docker-build-image.sh +++ b/tasks/scripts/docker-build-image.sh @@ -168,14 +168,19 @@ if [[ "${IS_FINAL_IMAGE}" == "1" ]]; then TAG_ARGS=(-t "${IMAGE_NAME}:${IMAGE_TAG}") fi +ATTESTATION_ARGS=(--provenance=false) OUTPUT_ARGS=() if [[ -n "${DOCKER_OUTPUT:-}" ]]; then OUTPUT_ARGS=(--output "${DOCKER_OUTPUT}") elif [[ "${IS_FINAL_IMAGE}" == "1" ]]; then - if [[ "${DOCKER_PUSH:-}" == "1" ]]; then - OUTPUT_ARGS=(--push) - elif [[ "${DOCKER_PLATFORM:-}" == *","* ]]; then - OUTPUT_ARGS=(--push) + if [[ "${DOCKER_PUSH:-}" == "1" || "${DOCKER_PLATFORM:-}" == *","* ]]; then + if ce_is_docker; then + # Attestations require a registry-backed image index. + ATTESTATION_ARGS=(--provenance=mode=min --attest type=sbom) + OUTPUT_ARGS=(--output "type=image,push=true,oci-mediatypes=true,oci-artifact=true") + else + OUTPUT_ARGS=(--push) + fi else OUTPUT_ARGS=(--load) fi @@ -191,7 +196,7 @@ ce_build \ -f "${DOCKERFILE}" \ --target "${DOCKER_TARGET}" \ ${TAG_ARGS[@]+"${TAG_ARGS[@]}"} \ - --provenance=false \ + ${ATTESTATION_ARGS[@]+"${ATTESTATION_ARGS[@]}"} \ "$@" \ ${OUTPUT_ARGS[@]+"${OUTPUT_ARGS[@]}"} \ . diff --git a/tasks/scripts/verify-image-sbom.sh b/tasks/scripts/verify-image-sbom.sh new file mode 100755 index 0000000000..1703b0608a --- /dev/null +++ b/tasks/scripts/verify-image-sbom.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Validate the final image's SBOM attestation and, for auditable builds, require +# at least one Cargo package discovered from the binary metadata. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + +usage() { + echo "Usage: verify-image-sbom.sh [--require-cargo]" >&2 +} + +IMAGE=${1:-} +REQUIRE_CARGO=${2:-} +if [[ -z "${IMAGE}" || $# -gt 2 || ( -n "${REQUIRE_CARGO}" && "${REQUIRE_CARGO}" != "--require-cargo" ) ]]; then + usage + exit 2 +fi + +if ! ce_is_docker; then + echo "Error: SBOM attestations are produced on the Docker/buildx path; ${CONTAINER_ENGINE} has no imagetools equivalent" >&2 + exit 2 +fi + +echo "==> Inspecting SBOM attestation of ${IMAGE}" +SBOM_JSON="$(ce buildx imagetools inspect "${IMAGE}" --format '{{ json .SBOM }}')" +COUNTS="$( + jq -r ' + [.. | objects | .SPDX? | select(type == "object")] as $documents + | [ + ($documents | length), + ([$documents[] | .. | strings | select(startswith("pkg:cargo/"))] | length) + ] + | @tsv + ' <<<"${SBOM_JSON}" +)" +read -r SPDX_COUNT CARGO_COUNT <<<"${COUNTS}" + +if [[ "${SPDX_COUNT}" -eq 0 ]]; then + echo "Error: ${IMAGE} carries no SPDX SBOM" >&2 + exit 1 +fi +if [[ "${REQUIRE_CARGO}" == "--require-cargo" && "${CARGO_COUNT}" -eq 0 ]]; then + echo "Error: ${IMAGE} SBOM contains no Cargo packages" >&2 + exit 1 +fi + +echo "SBOM attestation verified: SPDX=${SPDX_COUNT}, Cargo=${CARGO_COUNT}" From 981606d2f8e22a7e2a7d7c64f1bd5a797318c688 Mon Sep 17 00:00:00 2001 From: Simon Scatton <44714756+SDAChess@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:25:06 +0000 Subject: [PATCH 2/4] ci: build release binaries with Nix (#2977) * ci: build release binaries with Nix Refs #1683 Signed-off-by: Simon Scatton * ci: build VM artifacts with Nix Refs #1683 Signed-off-by: Simon Scatton * ci: build images from Nix artifacts Refs #1683 Signed-off-by: Simon Scatton * fix(nix): prevent host header leakage Signed-off-by: Simon Scatton * ci: parallelize artifact builds Refs #1683 Signed-off-by: Simon Scatton * ci: build external driver test artifacts Refs #1683 Signed-off-by: Simon Scatton * fix(nix): disable mold in musl shells Signed-off-by: Simon Scatton * ci: key Rust cache by Nix shell derivation Signed-off-by: Simon Scatton * ci: refactor end-to-end workflows Signed-off-by: Simon Scatton * ci: split platform binary workflows Signed-off-by: Simon Scatton * ci: remove obsolete native build workflows Signed-off-by: Simon Scatton * ci: replace disallowed mise action Signed-off-by: Simon Scatton * ci: fix refactored e2e lanes Signed-off-by: Simon Scatton * ci: check out local result action Signed-off-by: Simon Scatton * ci: cache mise installations Signed-off-by: Simon Scatton * ci: run docker builds on host runners Signed-off-by: Simon Scatton * ci: disable unstable kubernetes e2e lanes Signed-off-by: Simon Scatton * fix(ci): scope binary builds to cargo packages Signed-off-by: Simon Scatton * fix(ci): address zizmor template injection findings Signed-off-by: Simon Scatton * fix(ci): resolve remaining zizmor annotations Signed-off-by: Simon Scatton --------- Signed-off-by: Simon Scatton --- .github/actions/build-docker-image/action.yml | 75 +++ .github/actions/build-rust-binary/action.yml | 113 ++++ .github/actions/check-job-results/action.yml | 33 + .github/actions/setup-e2e-cli/action.yml | 12 +- .github/actions/setup-e2e-driver/action.yml | 33 + .github/actions/setup-e2e-gateway/action.yml | 14 +- .github/actions/setup-e2e-kind/action.yml | 82 +++ .github/actions/setup-e2e-podman/action.yml | 126 ++++ .../actions/setup-e2e-vm-driver/action.yml | 25 +- .github/actions/setup-e2e-vm/action.yml | 66 ++ .github/actions/setup-mise/action.yml | 42 ++ .github/workflows/branch-e2e.yml | 390 +++++++----- .github/workflows/build-binaries.yml | 78 +++ .github/workflows/build-cli-binaries.yml | 51 ++ .github/workflows/build-gateway-binaries.yml | 55 ++ .github/workflows/build-sandbox-binaries.yml | 48 ++ .github/workflows/build-vm-driver.yml | 102 ++++ .github/workflows/deb-package.yml | 23 +- .github/workflows/docker-build.yml | 324 ++-------- .github/workflows/driver-vm-linux.yml | 221 ------- .github/workflows/driver-vm-macos.yml | 221 ------- .github/workflows/e2e-docker-test.yml | 109 ++++ .github/workflows/e2e-gpu-test.yaml | 16 - .github/workflows/e2e-kubernetes-test.yml | 83 +-- .github/workflows/e2e-podman-test.yml | 91 +++ .github/workflows/e2e-test.yml | 234 ------- .github/workflows/e2e-vm-test.yml | 58 ++ .../workflows/package-release-binaries.yml | 77 +++ .github/workflows/release-dev.yml | 561 +++-------------- .github/workflows/release-tag.yml | 574 +++--------------- .github/workflows/rpm-package.yml | 14 +- .github/workflows/rust-native-build.yml | 383 ------------ .github/workflows/snap-package.yml | 29 +- .../workflows/supervisor-static-validate.yml | 70 --- CI.md | 10 +- architecture/build.md | 25 +- deploy/docker/Dockerfile.gateway | 15 +- deploy/docker/Dockerfile.supervisor | 24 +- e2e/rust/e2e-vm.sh | 2 +- e2e/support/gateway-common.sh | 13 +- e2e/with-kube-gateway.sh | 17 +- flake.nix | 7 + nix/devShells/musl.nix | 2 +- nix/pkgs/vm-runtime.nix | 46 ++ nix/toolchains/linux-gnu-2.28/default.nix | 5 +- scripts/baseline_workflow_metrics.py | 16 +- 46 files changed, 1909 insertions(+), 2706 deletions(-) create mode 100644 .github/actions/build-docker-image/action.yml create mode 100644 .github/actions/build-rust-binary/action.yml create mode 100644 .github/actions/check-job-results/action.yml create mode 100644 .github/actions/setup-e2e-driver/action.yml create mode 100644 .github/actions/setup-e2e-kind/action.yml create mode 100644 .github/actions/setup-e2e-podman/action.yml create mode 100644 .github/actions/setup-e2e-vm/action.yml create mode 100644 .github/actions/setup-mise/action.yml create mode 100644 .github/workflows/build-binaries.yml create mode 100644 .github/workflows/build-cli-binaries.yml create mode 100644 .github/workflows/build-gateway-binaries.yml create mode 100644 .github/workflows/build-sandbox-binaries.yml create mode 100644 .github/workflows/build-vm-driver.yml delete mode 100644 .github/workflows/driver-vm-linux.yml delete mode 100644 .github/workflows/driver-vm-macos.yml create mode 100644 .github/workflows/e2e-docker-test.yml create mode 100644 .github/workflows/e2e-podman-test.yml delete mode 100644 .github/workflows/e2e-test.yml create mode 100644 .github/workflows/e2e-vm-test.yml create mode 100644 .github/workflows/package-release-binaries.yml delete mode 100644 .github/workflows/rust-native-build.yml delete mode 100644 .github/workflows/supervisor-static-validate.yml create mode 100644 nix/pkgs/vm-runtime.nix diff --git a/.github/actions/build-docker-image/action.yml b/.github/actions/build-docker-image/action.yml new file mode 100644 index 0000000000..08557fbfd4 --- /dev/null +++ b/.github/actions/build-docker-image/action.yml @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Build Docker Image +description: Build and push one architecture of an OpenShell image + +inputs: + component: + description: Docker image and Dockerfile name + required: true + binary: + description: Binary staged in the Docker build context + required: true + triple: + description: Binary artifact target triple + required: true + arch: + description: Docker architecture name + required: true + platform: + description: Docker platform + required: true + image-tag: + description: Docker image tag + required: true + github-token: + description: Token used to push the image + required: true + +runs: + using: composite + steps: + - uses: ./.github/actions/setup-buildx + with: + buildkitd-config: /etc/buildkit/buildkitd.toml + + - name: Log in to GHCR + shell: bash + run: echo "${INPUTS_GITHUB_TOKEN}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + env: + INPUTS_GITHUB_TOKEN: ${{ inputs.github-token }} + + - name: Download ${{ inputs.binary }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.binary }}-${{ inputs.triple }} + path: artifact + + - name: Stage ${{ inputs.binary }} + shell: bash + run: install -Dm0755 artifact/${INPUTS_BINARY} deploy/docker/.build/prebuilt-binaries/${INPUTS_ARCH}/${INPUTS_BINARY} + env: + INPUTS_BINARY: ${{ inputs.binary }} + INPUTS_ARCH: ${{ inputs.arch }} + + - name: Build ${{ inputs.component }} image + shell: bash + env: + IMAGE_TAG: ${{ inputs.image-tag }} + INPUTS_PLATFORM: ${{ inputs.platform }} + INPUTS_COMPONENT: ${{ inputs.component }} + INPUTS_ARCH: ${{ inputs.arch }} + run: | + docker buildx build \ + --builder openshell \ + --platform ${INPUTS_PLATFORM} \ + --file deploy/docker/Dockerfile.${INPUTS_COMPONENT} \ + --target ${INPUTS_COMPONENT} \ + --tag ghcr.io/nvidia/openshell/${INPUTS_COMPONENT}:${IMAGE_TAG}-${INPUTS_ARCH} \ + --cache-from type=gha,scope=${INPUTS_COMPONENT}-${INPUTS_ARCH} \ + --cache-to type=gha,mode=max,scope=${INPUTS_COMPONENT}-${INPUTS_ARCH} \ + --provenance=mode=min \ + --attest type=sbom \ + --output type=image,push=true,oci-mediatypes=true,oci-artifact=true \ + . diff --git a/.github/actions/build-rust-binary/action.yml b/.github/actions/build-rust-binary/action.yml new file mode 100644 index 0000000000..98ba51b88e --- /dev/null +++ b/.github/actions/build-rust-binary/action.yml @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Build Rust Binary +description: Build, verify, package, and upload an auditable Rust binary + +inputs: + package: + description: Cargo package that owns the binary + required: true + binary: + description: Cargo binary name + required: true + triple: + description: Rust target triple + required: true + dev-shell: + description: Nix development shell used to build the binary + required: true + cargo-version: + description: Cargo package version embedded in the binary + required: true + image-tag: + description: Default supervisor image tag embedded in the binary + required: false + default: "" + artifact-name: + description: GitHub artifact name + required: false + default: "" + extra-cargo-flags: + description: Additional flags passed to cargo build + required: false + default: "" + interpreter: + description: ELF interpreter for a dynamically linked Linux binary + required: false + default: "" + +runs: + using: composite + steps: + - name: Hash development shell + id: dev-shell + shell: bash + env: + DEV_SHELL: ${{ inputs.dev-shell }} + run: echo "hash=$(nix hash file --type sha256 --base16 "$(nix eval --raw "${DEV_SHELL}.drvPath")")" >> "$GITHUB_OUTPUT" + + - name: Cache Rust artifacts + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: binaries-${{ inputs.binary }}-${{ inputs.triple }}-${{ steps.dev-shell.outputs.hash }} + cache-on-failure: "true" + cache-workspace-crates: "true" + cache-bin: "false" + cmd-format: nix develop ${{ inputs.dev-shell }} -c {0} + + - name: Set version + shell: nix develop ${{ inputs.dev-shell }} -c bash -euo pipefail {0} + env: + INPUTS_CARGO_VERSION: ${{ inputs.cargo-version }} + run: sed -i "s/^version = \"0\\.0\\.0\"$/version = \"${INPUTS_CARGO_VERSION}\"/" Cargo.toml + + - name: Build ${{ inputs.binary }} + shell: nix develop ${{ inputs.dev-shell }} -c bash -euo pipefail {0} + env: + OPENSHELL_IMAGE_TAG: ${{ inputs.image-tag }} + INPUTS_PACKAGE: ${{ inputs.package }} + INPUTS_BINARY: ${{ inputs.binary }} + INPUTS_EXTRA_CARGO_FLAGS: ${{ inputs.extra-cargo-flags }} + run: GIT_DIR=/nonexistent cargo auditable build --release --package "${INPUTS_PACKAGE}" --bin "${INPUTS_BINARY}" ${INPUTS_EXTRA_CARGO_FLAGS} + + - name: Verify ${{ inputs.binary }} + shell: nix develop ${{ inputs.dev-shell }} -c bash -euo pipefail {0} + env: + INPUTS_BINARY: ${{ inputs.binary }} + run: | + # Confirm the binary runs and reports the expected name. + target/release/${INPUTS_BINARY} --version | grep -q "^${INPUTS_BINARY} " + # Confirm Syft can decode the embedded cargo-auditable metadata. + SYFT_CHECK_FOR_APP_UPDATE=false syft file:target/release/${INPUTS_BINARY} -o cyclonedx-json | grep 'pkg:cargo/' > /dev/null + + - name: Verify static linkage + if: endsWith(inputs.triple, '-linux-musl') + shell: nix develop ${{ inputs.dev-shell }} -c bash -euo pipefail {0} + env: + INPUTS_BINARY: ${{ inputs.binary }} + run: tasks/scripts/verify-static-binary.sh target/release/${INPUTS_BINARY} + + - name: Normalize Linux dynamic binary + if: endsWith(inputs.triple, '-linux-gnu') + shell: nix develop ${{ inputs.dev-shell }} -c bash -euo pipefail {0} + env: + INPUTS_BINARY: ${{ inputs.binary }} + INPUTS_INTERPRETER: ${{ inputs.interpreter }} + run: | + binary=target/release/${INPUTS_BINARY} + # Remove Nix store paths so the binary can run on other distributions. + patchelf --set-interpreter "${INPUTS_INTERPRETER}" --remove-rpath "$binary" + # Z3 must be embedded instead of loaded from the target system. + test -z "$(patchelf --print-needed "$binary" | grep '^libz3')" + # Reject symbols introduced after glibc 2.28. + tasks/scripts/verify-glibc-symbols.sh 2.28 "$binary" + + - name: Upload ${{ inputs.binary }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.artifact-name || format('{0}-{1}', inputs.binary, inputs.triple) }} + path: target/release/${{ inputs.binary }} + compression-level: 0 + retention-days: 5 + if-no-files-found: error diff --git a/.github/actions/check-job-results/action.yml b/.github/actions/check-job-results/action.yml new file mode 100644 index 0000000000..bd6beb9456 --- /dev/null +++ b/.github/actions/check-job-results/action.yml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Check job results +description: Fail when any required upstream job did not succeed + +inputs: + results: + description: JSON-encoded GitHub Actions needs context + required: true + +runs: + using: composite + steps: + - name: Check required jobs + shell: bash + env: + JOB_RESULTS: ${{ inputs.results }} + run: | + set -euo pipefail + failures="$( + jq -r ' + to_entries[] + | select(.value.result != "success") + | "\(.key) concluded \(.value.result)" + ' <<< "$JOB_RESULTS" + )" + if [ -n "$failures" ]; then + while IFS= read -r failure; do + echo "::error::$failure" + done <<< "$failures" + exit 1 + fi diff --git a/.github/actions/setup-e2e-cli/action.yml b/.github/actions/setup-e2e-cli/action.yml index fb894194c8..4d43e232a4 100644 --- a/.github/actions/setup-e2e-cli/action.yml +++ b/.github/actions/setup-e2e-cli/action.yml @@ -1,29 +1,19 @@ name: Setup E2E CLI description: Download an architecture-matched prebuilt OpenShell CLI for E2E tests -inputs: - artifact-prefix: - description: Artifact name prefix; linux- is appended automatically - required: true - runs: using: composite steps: - name: Download prebuilt CLI uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ format('{0}-linux-{1}', inputs.artifact-prefix, runner.arch == 'X64' && 'amd64' || 'arm64') }} + name: ${{ runner.arch == 'X64' && 'openshell-x86_64-unknown-linux-musl' || 'openshell-aarch64-unknown-linux-musl' }} path: .e2e/prebuilt-cli - name: Configure prebuilt CLI shell: bash run: | - set -euo pipefail cli="$GITHUB_WORKSPACE/.e2e/prebuilt-cli/openshell" - if [[ ! -f "$cli" ]]; then - echo "downloaded artifact is missing $cli" >&2 - exit 1 - fi chmod +x "$cli" "$cli" --version echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV" diff --git a/.github/actions/setup-e2e-driver/action.yml b/.github/actions/setup-e2e-driver/action.yml new file mode 100644 index 0000000000..c2172b2a63 --- /dev/null +++ b/.github/actions/setup-e2e-driver/action.yml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Setup E2E Driver +description: Download an architecture-matched standalone compute driver for E2E tests + +inputs: + binary: + description: Compute driver binary name + required: true + +runs: + using: composite + steps: + - name: Download prebuilt ${{ inputs.binary }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.binary }}-${{ runner.arch == 'X64' && 'x86_64-unknown-linux-gnu' || 'aarch64-unknown-linux-gnu' }} + path: .e2e/prebuilt-driver + + - name: Configure prebuilt ${{ inputs.binary }} + shell: bash + run: | # zizmor: ignore[github-env] validated filename under the trusted workspace path + if [[ ! "${INPUTS_BINARY}" =~ ^[a-zA-Z0-9._-]+$ ]]; then + echo "invalid driver binary name: ${INPUTS_BINARY}" >&2 + exit 1 + fi + driver="$GITHUB_WORKSPACE/.e2e/prebuilt-driver/${INPUTS_BINARY}" + chmod +x "$driver" + "$driver" --version + echo "OPENSHELL_EXTERNAL_DRIVER_BIN=$driver" >> "$GITHUB_ENV" + env: + INPUTS_BINARY: ${{ inputs.binary }} diff --git a/.github/actions/setup-e2e-gateway/action.yml b/.github/actions/setup-e2e-gateway/action.yml index b0ee52392b..255a7f4e4c 100644 --- a/.github/actions/setup-e2e-gateway/action.yml +++ b/.github/actions/setup-e2e-gateway/action.yml @@ -2,9 +2,10 @@ name: Setup E2E Gateway description: Download an architecture-matched prebuilt OpenShell gateway for E2E tests inputs: - artifact-prefix: - description: Artifact name prefix; linux- is appended automatically - required: true + artifact-name: + description: GitHub artifact name + required: false + default: "" runs: using: composite @@ -12,18 +13,13 @@ runs: - name: Download prebuilt gateway uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ format('{0}-linux-{1}', inputs.artifact-prefix, runner.arch == 'X64' && 'amd64' || 'arm64') }} + name: ${{ inputs.artifact-name || (runner.arch == 'X64' && 'openshell-gateway-x86_64-unknown-linux-gnu' || 'openshell-gateway-aarch64-unknown-linux-gnu') }} path: .e2e/prebuilt-gateway - name: Configure prebuilt gateway shell: bash run: | - set -euo pipefail gateway="$GITHUB_WORKSPACE/.e2e/prebuilt-gateway/openshell-gateway" - if [[ ! -f "$gateway" ]]; then - echo "downloaded artifact is missing $gateway" >&2 - exit 1 - fi chmod +x "$gateway" "$gateway" --version echo "OPENSHELL_GATEWAY_BIN=$gateway" >> "$GITHUB_ENV" diff --git a/.github/actions/setup-e2e-kind/action.yml b/.github/actions/setup-e2e-kind/action.yml new file mode 100644 index 0000000000..b9fa8375ba --- /dev/null +++ b/.github/actions/setup-e2e-kind/action.yml @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Setup E2E kind +description: Create a kind cluster and preload OpenShell images for E2E tests + +inputs: + cluster-name: + description: kind cluster name + required: true + image-tag: + description: OpenShell image tag to load + required: true + images: + description: Space-separated OpenShell image components to load + required: false + default: gateway supervisor + registry: + description: Container registry and namespace + required: false + default: ghcr.io/nvidia/openshell + registry-username: + description: Registry username + required: true + registry-password: + description: Registry password + required: true + mise-version: + description: mise release to install + required: false + default: v2026.4.25 + +runs: + using: composite + steps: + - uses: ./.github/actions/setup-mise + with: + version: ${{ inputs.mise-version }} + + - name: Log in to GHCR + shell: bash + env: + REGISTRY: ${{ inputs.registry }} + REGISTRY_USERNAME: ${{ inputs.registry-username }} + REGISTRY_PASSWORD: ${{ inputs.registry-password }} + run: echo "$REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$REGISTRY_USERNAME" --password-stdin + + - name: Create kind cluster + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 + with: + cluster_name: ${{ inputs.cluster-name }} + wait: 120s + + - name: Export kind kubeconfig to mise path + shell: bash + env: + CLUSTER_NAME: ${{ inputs.cluster-name }} + run: | + set -euo pipefail + kind get kubeconfig --name "$CLUSTER_NAME" > "$GITHUB_WORKSPACE/kubeconfig" + chmod 600 "$GITHUB_WORKSPACE/kubeconfig" + + - name: Load OpenShell images into kind + shell: bash + env: + CLUSTER_NAME: ${{ inputs.cluster-name }} + IMAGE_COMPONENTS: ${{ inputs.images }} + IMAGE_TAG: ${{ inputs.image-tag }} + REGISTRY: ${{ inputs.registry }} + run: | + set -euo pipefail + for component in $IMAGE_COMPONENTS; do + case "$component" in + gateway | supervisor) ;; + *) echo "ERROR: unsupported OpenShell image component: $component" >&2; exit 1 ;; + esac + image="${REGISTRY}/${component}:${IMAGE_TAG}" + archive="${RUNNER_TEMP:-/tmp}/openshell-${component}-linux-amd64.tar" + docker pull --platform linux/amd64 "$image" + docker image save --platform linux/amd64 --output "$archive" "$image" + kind load image-archive "$archive" --name "$CLUSTER_NAME" + done diff --git a/.github/actions/setup-e2e-podman/action.yml b/.github/actions/setup-e2e-podman/action.yml new file mode 100644 index 0000000000..47f57f5be8 --- /dev/null +++ b/.github/actions/setup-e2e-podman/action.yml @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Setup E2E Podman +description: Configure the supported rootless Podman environment for E2E tests + +inputs: + mise-version: + description: mise release to install + required: false + default: v2026.4.25 + podman-major: + description: Expected Podman major version + required: true + podman-package-version: + description: Exact Ubuntu Podman package version + required: true + conmon-package-version: + description: Exact Ubuntu conmon package version + required: true + +runs: + using: composite + steps: + - uses: ./.github/actions/setup-mise + with: + version: ${{ inputs.mise-version }} + + - name: Install Podman and build dependencies + shell: bash + run: | # zizmor: ignore[github-env] persists only literal and trusted runner-derived paths + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + apparmor \ + build-essential \ + fuse-overlayfs \ + libssl-dev \ + openssh-client \ + passt \ + pkg-config \ + "conmon=${INPUTS_CONMON_PACKAGE_VERSION}" \ + "podman=${INPUTS_PODMAN_PACKAGE_VERSION}" \ + uidmap + # Hosted runners can place newer binaries under /usr/local. Select the + # distro CLI and use Podman's supported final conmon-path override. + podman_config="${RUNNER_TEMP}/openshell-containers.conf" + printf '%s\n' \ + '[engine]' \ + 'conmon_path = ["/usr/bin/conmon"]' \ + > "${podman_config}" + echo "/usr/bin" >> "${GITHUB_PATH}" + echo "CONTAINERS_CONF_OVERRIDE=${podman_config}" >> "${GITHUB_ENV}" + env: + INPUTS_CONMON_PACKAGE_VERSION: ${{ inputs.conmon-package-version }} + INPUTS_PODMAN_PACKAGE_VERSION: ${{ inputs.podman-package-version }} + + - name: Allow pasta to receive Podman stop signals + shell: bash + run: | + set -euo pipefail + # Ubuntu's packaged profile blocks this signal and makes Podman wait + # for its SIGKILL fallback. + profile=/etc/apparmor.d/usr.bin.pasta + rule=' signal (receive) peer=podman,' + if ! sudo grep -Fqx "${rule}" "${profile}"; then + sudo sed -i '\|^ include $|a\ signal (receive) peer=podman,' "${profile}" + fi + sudo grep -Fqx "${rule}" "${profile}" + sudo apparmor_parser --replace "${profile}" + + - name: Configure rootless Podman + shell: bash + run: | # zizmor: ignore[github-env] runtime path is derived solely from the runner UID + set -euo pipefail + if ! grep -q "^${USER}:" /etc/subuid; then + sudo usermod --add-subuids 100000-165535 "$USER" + fi + if ! grep -q "^${USER}:" /etc/subgid; then + sudo usermod --add-subgids 100000-165535 "$USER" + fi + runtime_dir="/run/user/$(id -u)" + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "$runtime_dir" + echo "XDG_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" + + - name: Verify rootless Podman environment + shell: bash + run: | + set -euo pipefail + podman_version="$(podman version --format '{{.Client.Version}}')" + case "$podman_version" in + "${INPUTS_PODMAN_MAJOR}".*) ;; + *) echo "ERROR: expected Podman ${INPUTS_PODMAN_MAJOR}.x, found $podman_version" >&2; exit 1 ;; + esac + test "$(dpkg-query -W -f='${Version}' podman)" = "${INPUTS_PODMAN_PACKAGE_VERSION}" + test "$(dpkg-query -W -f='${Version}' conmon)" = "${INPUTS_CONMON_PACKAGE_VERSION}" + test "$(command -v podman)" = "/usr/bin/podman" + test "$(podman info --format '{{.Host.Conmon.Path}}')" = "/usr/bin/conmon" + test "$(podman info --format '{{.Host.Security.Rootless}}')" = "true" + test "$(podman info --format '{{.Host.RootlessNetworkCmd}}')" = "pasta" + test "$(sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns)" = "1" + echo "=== host ===" + uname -a + echo "=== AppArmor ===" + cat /proc/self/attr/current + sudo aa-status || true + echo "=== Podman ===" + podman version + podman info --debug + env: + INPUTS_PODMAN_MAJOR: ${{ inputs.podman-major }} + INPUTS_PODMAN_PACKAGE_VERSION: ${{ inputs.podman-package-version }} + INPUTS_CONMON_PACKAGE_VERSION: ${{ inputs.conmon-package-version }} + + - name: Probe rootless capability bounding set + shell: bash + run: | + set -euo pipefail + probe="$RUNNER_TEMP/openshell-capbset-probe" + cc -static -O2 -Wall -Wextra -Werror \ + e2e/support/capbset-probe.c \ + -o "$probe" + podman run --rm \ + --cap-add=SETPCAP \ + --volume "$probe:/openshell-capbset-probe:ro" \ + docker.io/library/alpine:3.22 \ + /openshell-capbset-probe diff --git a/.github/actions/setup-e2e-vm-driver/action.yml b/.github/actions/setup-e2e-vm-driver/action.yml index 20c43cf98f..a074ed97a8 100644 --- a/.github/actions/setup-e2e-vm-driver/action.yml +++ b/.github/actions/setup-e2e-vm-driver/action.yml @@ -1,10 +1,5 @@ name: Setup E2E VM Driver -description: Download a prebuilt OpenShell VM driver for E2E tests - -inputs: - artifact-name: - description: Artifact name to download, such as driver-vm-linux-amd64 - required: true +description: Download an architecture-matched prebuilt OpenShell VM driver for E2E tests runs: using: composite @@ -12,27 +7,13 @@ runs: - name: Download prebuilt VM driver uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ inputs.artifact-name }} + name: ${{ runner.arch == 'X64' && 'openshell-driver-vm-x86_64-unknown-linux-gnu' || 'openshell-driver-vm-aarch64-unknown-linux-gnu' }} path: .e2e/prebuilt-vm-driver - name: Configure prebuilt VM driver shell: bash run: | - set -euo pipefail - extract_dir="$GITHUB_WORKSPACE/.e2e/prebuilt-vm-driver/bin" - mkdir -p "$extract_dir" - archive="$(find "$GITHUB_WORKSPACE/.e2e/prebuilt-vm-driver" -maxdepth 1 -type f -name 'openshell-driver-vm-*.tar.gz' -print -quit)" - if [[ -z "$archive" ]]; then - echo "downloaded artifact is missing openshell-driver-vm-*.tar.gz" >&2 - find "$GITHUB_WORKSPACE/.e2e/prebuilt-vm-driver" -maxdepth 2 -type f -print >&2 || true - exit 1 - fi - tar -xzf "$archive" -C "$extract_dir" - driver="$extract_dir/openshell-driver-vm" - if [[ ! -f "$driver" ]]; then - echo "downloaded artifact is missing $driver" >&2 - exit 1 - fi + driver="$GITHUB_WORKSPACE/.e2e/prebuilt-vm-driver/openshell-driver-vm" chmod +x "$driver" "$driver" --version echo "OPENSHELL_VM_DRIVER_BIN=$driver" >> "$GITHUB_ENV" diff --git a/.github/actions/setup-e2e-vm/action.yml b/.github/actions/setup-e2e-vm/action.yml new file mode 100644 index 0000000000..6164d999c5 --- /dev/null +++ b/.github/actions/setup-e2e-vm/action.yml @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Setup E2E VM +description: Configure a bare Linux KVM host for VM E2E tests + +inputs: + mise-version: + description: mise release to install + required: false + default: v2026.4.25 + rust-cache-key: + description: Shared key for Rust test artifacts + required: false + default: e2e-vm-linux-amd64 + +runs: + using: composite + steps: + - name: Install system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + libssl-dev \ + openssh-client \ + pkg-config \ + socat \ + zstd + + - name: Enable KVM access + shell: bash + run: | + set -euo pipefail + if [[ ! -c /dev/kvm ]]; then + echo "::error::The GitHub-hosted runner did not expose /dev/kvm" + lscpu + grep -m1 -E '^(flags|Features)' /proc/cpuinfo || true + ls -la /dev + exit 1 + fi + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --settle --name-match=kvm + ls -l /dev/kvm + exec 3<>/dev/kvm + exec 3>&- + + - name: Validate VM host tools + shell: bash + run: | + command -v mke2fs + command -v mkfs.ext4 + command -v debugfs + + - uses: ./.github/actions/setup-mise + with: + version: ${{ inputs.mise-version }} + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: ${{ inputs.rust-cache-key }} + cache-on-failure: "true" diff --git a/.github/actions/setup-mise/action.yml b/.github/actions/setup-mise/action.yml new file mode 100644 index 0000000000..e5a94a6801 --- /dev/null +++ b/.github/actions/setup-mise/action.yml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Setup mise +description: Install mise and the tools pinned by the repository + +inputs: + version: + description: mise release to install + required: false + default: v2026.4.25 + +runs: + using: composite + steps: + - name: Restore mise cache + id: mise-cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ~/.local/share/mise + key: mise-${{ runner.os }}-${{ runner.arch }}-${{ inputs.version }}-${{ hashFiles('mise.toml', 'mise.lock') }} + restore-keys: | + mise-${{ runner.os }}-${{ runner.arch }}-${{ inputs.version }}- + + - name: Install mise and tools + shell: bash + env: + MISE_VERSION: ${{ inputs.version }} + run: | # zizmor: ignore[github-env] PATH entries are fixed beneath the runner-controlled HOME + set -euo pipefail + curl https://mise.run | sh + export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" + mise install --locked + + - name: Save mise cache + if: steps.mise-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: ~/.local/share/mise + key: ${{ steps.mise-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 659027a8d6..0b050d0e51 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -44,8 +44,8 @@ jobs: push) run_core_e2e="$(jq -r 'index("test:e2e") != null' <<< "$LABELS_JSON")" run_gpu_e2e="$(jq -r 'index("test:e2e-gpu") != null' <<< "$LABELS_JSON")" - run_kubernetes_ha_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")" - run_kubernetes_credential_drivers_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")" + run_kubernetes_ha_e2e=false + run_kubernetes_credential_drivers_e2e=false ;; merge_group) # Merge groups have no PR labels. When GPU E2E is required as documented @@ -59,8 +59,8 @@ jobs: *) run_core_e2e=true run_gpu_e2e=true - run_kubernetes_ha_e2e=true - run_kubernetes_credential_drivers_e2e=true + run_kubernetes_ha_e2e=false + run_kubernetes_credential_drivers_e2e=false ;; esac if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ] || [ "$run_kubernetes_credential_drivers_e2e" = "true" ]; then @@ -76,20 +76,159 @@ jobs: echo "run_any_e2e=$run_any_e2e" } >> "$GITHUB_OUTPUT" - build-gateway: + version: needs: [pr_metadata] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true' + permissions: + contents: read + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + cargo: ${{ steps.version.outputs.cargo }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Compute version + id: version + run: | + cargo="$(python3 tasks/scripts/release.py get-version --cargo)" + echo "cargo=$cargo" >> "$GITHUB_OUTPUT" + + build-cli: + needs: version + permissions: + contents: read + uses: ./.github/workflows/build-cli-binaries.yml + with: + cargo-version: ${{ needs.version.outputs.cargo }} + secrets: inherit + + build-sandbox: + needs: version + permissions: + contents: read + uses: ./.github/workflows/build-sandbox-binaries.yml + with: + cargo-version: ${{ needs.version.outputs.cargo }} + secrets: inherit + + build-gateway: + needs: version + permissions: + contents: read + uses: ./.github/workflows/build-gateway-binaries.yml + with: + cargo-version: ${{ needs.version.outputs.cargo }} + secrets: inherit + + build-gateway-plain: + needs: [pr_metadata, version] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + contents: read + strategy: + matrix: + include: + - triple: x86_64-unknown-linux-gnu + runner: linux-amd64-cpu8 + dev_shell: .#devShells.x86_64-linux.glibc-2-28 + interpreter: /lib64/ld-linux-x86-64.so.2 + - triple: aarch64-unknown-linux-gnu + runner: linux-arm64-cpu8 + dev_shell: .#devShells.aarch64-linux.glibc-2-28 + interpreter: /lib/ld-linux-aarch64.so.1 + uses: ./.github/workflows/build-binaries.yml + with: + package: openshell-server + binary: openshell-gateway + artifact-name: openshell-gateway-plain-${{ matrix.triple }} + triple: ${{ matrix.triple }} + runner: ${{ matrix.runner }} + dev-shell: ${{ matrix.dev_shell }} + cargo-version: ${{ needs.version.outputs.cargo }} + extra-cargo-flags: --no-default-features + interpreter: ${{ matrix.interpreter }} + secrets: inherit + + build-driver-docker: + needs: [pr_metadata, version] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + contents: read + uses: ./.github/workflows/build-binaries.yml + with: + package: openshell-driver-docker + binary: openshell-driver-docker + triple: aarch64-unknown-linux-gnu + runner: linux-arm64-cpu8 + dev-shell: .#devShells.aarch64-linux.glibc-2-28 + cargo-version: ${{ needs.version.outputs.cargo }} + interpreter: /lib/ld-linux-aarch64.so.1 + secrets: inherit + + build-driver-podman: + needs: [pr_metadata, version] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + contents: read + uses: ./.github/workflows/build-binaries.yml + with: + package: openshell-driver-podman + binary: openshell-driver-podman + triple: x86_64-unknown-linux-gnu + runner: linux-amd64-cpu8 + dev-shell: .#devShells.x86_64-linux.glibc-2-28 + cargo-version: ${{ needs.version.outputs.cargo }} + interpreter: /lib64/ld-linux-x86-64.so.2 + secrets: inherit + + build-driver-kubernetes: + needs: [pr_metadata, version] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + contents: read + uses: ./.github/workflows/build-binaries.yml + with: + package: openshell-driver-kubernetes + binary: openshell-driver-kubernetes + triple: x86_64-unknown-linux-gnu + runner: linux-amd64-cpu8 + dev-shell: .#devShells.x86_64-linux.glibc-2-28 + cargo-version: ${{ needs.version.outputs.cargo }} + interpreter: /lib64/ld-linux-x86-64.so.2 + secrets: inherit + + build-vm-driver: + needs: [pr_metadata, version, build-sandbox] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + contents: read + uses: ./.github/workflows/build-vm-driver.yml + with: + cargo-version: ${{ needs.version.outputs.cargo }} + secrets: inherit + + build-gateway-image: + needs: [pr_metadata, build-gateway] + if: >- + needs.pr_metadata.outputs.should_run == 'true' && + (needs.pr_metadata.outputs.run_core_e2e == 'true' || + needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' || + needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true') permissions: contents: read packages: write uses: ./.github/workflows/docker-build.yml with: component: gateway - image-tag: ${{ github.sha }} - auditable: true + binary: openshell-gateway + target-suffix: unknown-linux-gnu + secrets: inherit - build-supervisor: - needs: [pr_metadata] + build-supervisor-image: + needs: [pr_metadata, build-sandbox] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true' permissions: contents: read @@ -97,51 +236,88 @@ jobs: uses: ./.github/workflows/docker-build.yml with: component: supervisor - image-tag: ${{ github.sha }} - auditable: true + binary: openshell-sandbox + target-suffix: unknown-linux-musl + secrets: inherit - build-cli: - needs: [pr_metadata] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true' + docker-e2e: + needs: [pr_metadata, build-cli, build-gateway, build-supervisor-image] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: + actions: read contents: read - packages: write - uses: ./.github/workflows/docker-build.yml + packages: read + uses: ./.github/workflows/e2e-docker-test.yml with: - component: cli - platform: linux/amd64,linux/arm64 + image-tag: ${{ github.sha }} + runner: linux-arm64-cpu8 - build-driver-vm-linux: - needs: [pr_metadata] + podman-e2e: + needs: [pr_metadata, build-cli, build-gateway, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: + actions: read contents: read packages: read - uses: ./.github/workflows/driver-vm-linux.yml + uses: ./.github/workflows/e2e-podman-test.yml with: image-tag: ${{ github.sha }} - checkout-ref: ${{ github.sha }} - runtime-platforms: linux-x86_64 - driver-targets: >- - [{"arch":"amd64","runner":"linux-amd64-cpu8","target":"x86_64-unknown-linux-gnu","zig_target":"x86_64-unknown-linux-gnu.2.28","platform":"linux-x86_64","guest_arch":"x86_64"}] - e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux] + vm-e2e: + needs: [pr_metadata, build-cli, build-gateway, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-test.yml + uses: ./.github/workflows/e2e-vm-test.yml + + docker-external-driver-e2e: + needs: [pr_metadata, build-cli, build-gateway-plain, build-driver-docker, build-supervisor-image] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-docker-test.yml with: image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 - cli-artifact-prefix: rust-binary-cli - gateway-artifact-prefix: rust-binary-gateway - vm-driver-artifact-name: driver-vm-linux-amd64 + gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu + external-driver-binary: openshell-driver-docker + suite-matrix: >- + [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] + + podman-external-driver-e2e: + needs: [pr_metadata, build-cli, build-gateway-plain, build-driver-podman, build-supervisor-image] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-podman-test.yml + with: + image-tag: ${{ github.sha }} + gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu + external-driver-binary: openshell-driver-podman + suite-matrix: >- + [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] + + vm-external-driver-e2e: + needs: [pr_metadata, build-cli, build-gateway-plain, build-vm-driver] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-vm-test.yml + with: + gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu + suite-name: external-driver + e2e-task: e2e:vm:external-driver gpu-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-cli, build-gateway, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true' permissions: actions: read @@ -150,11 +326,9 @@ jobs: uses: ./.github/workflows/e2e-gpu-test.yaml with: image-tag: ${{ github.sha }} - cli-artifact-prefix: rust-binary-cli - gateway-artifact-prefix: rust-binary-gateway kubernetes-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' strategy: fail-fast: false @@ -182,10 +356,9 @@ jobs: job-name: Kubernetes E2E (Rust smoke, ${{ matrix.topology }}, Agent Sandbox ${{ matrix.agent_sandbox_api }}) agent-sandbox-version: ${{ matrix.agent_sandbox_version }} extra-helm-values: ${{ matrix.extra_helm_values }} - cli-artifact-prefix: rust-binary-cli kubernetes-workspace-managed-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -196,10 +369,9 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace managed mode) e2e-task: e2e:kubernetes:workspace-managed - cli-artifact-prefix: rust-binary-cli kubernetes-external-driver-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-cli, build-gateway-plain, build-driver-kubernetes, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -210,10 +382,12 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (external compute driver) e2e-task: e2e:kubernetes:external-driver - cli-artifact-prefix: rust-binary-cli + gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu + external-driver-binary: openshell-driver-kubernetes + cluster-images: supervisor kubernetes-workspace-operator-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -224,10 +398,9 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace operator mode) e2e-task: e2e:kubernetes:workspace-operator - cli-artifact-prefix: rust-binary-cli kubernetes-ha-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' permissions: actions: read @@ -239,10 +412,9 @@ jobs: job-name: Kubernetes HA E2E (Rust smoke) extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml external-postgres-secret: openshell-ha-pg - cli-artifact-prefix: rust-binary-cli kubernetes-credential-drivers-e2e: - needs: [pr_metadata, build-gateway, build-supervisor] + needs: [pr_metadata, build-cli, build-gateway-image, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' permissions: actions: read @@ -256,124 +428,52 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, docker-e2e, podman-e2e, vm-e2e, docker-external-driver-e2e, podman-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: - - name: Verify core E2E jobs - env: - BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }} - BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} - BUILD_CLI_RESULT: ${{ needs.build-cli.result }} - BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }} - E2E_RESULT: ${{ needs.e2e.result }} - KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }} - KUBERNETES_EXTERNAL_DRIVER_E2E_RESULT: ${{ needs.kubernetes-external-driver-e2e.result }} - KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT: ${{ needs.kubernetes-workspace-managed-e2e.result }} - KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT: ${{ needs.kubernetes-workspace-operator-e2e.result }} - run: | - set -euo pipefail - failed=0 - for item in \ - "build-gateway:$BUILD_GATEWAY_RESULT" \ - "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ - "build-cli:$BUILD_CLI_RESULT" \ - "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \ - "e2e:$E2E_RESULT" \ - "kubernetes-e2e:$KUBERNETES_E2E_RESULT" \ - "kubernetes-external-driver-e2e:$KUBERNETES_EXTERNAL_DRIVER_E2E_RESULT" \ - "kubernetes-workspace-managed-e2e:$KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT" \ - "kubernetes-workspace-operator-e2e:$KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT"; do - name="${item%%:*}" - result="${item#*:}" - if [ "$result" != "success" ]; then - echo "::error::$name concluded $result" - failed=1 - fi - done - exit "$failed" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/check-job-results + with: + results: ${{ toJSON(needs) }} gpu-e2e-result: name: GPU E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, gpu-e2e] + needs: [pr_metadata, gpu-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true' runs-on: ubuntu-latest steps: - - name: Verify GPU E2E jobs - env: - BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }} - BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} - BUILD_CLI_RESULT: ${{ needs.build-cli.result }} - GPU_E2E_RESULT: ${{ needs.gpu-e2e.result }} - run: | - set -euo pipefail - failed=0 - for item in \ - "build-gateway:$BUILD_GATEWAY_RESULT" \ - "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ - "build-cli:$BUILD_CLI_RESULT" \ - "gpu-e2e:$GPU_E2E_RESULT"; do - name="${item%%:*}" - result="${item#*:}" - if [ "$result" != "success" ]; then - echo "::error::$name concluded $result" - failed=1 - fi - done - exit "$failed" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/check-job-results + with: + results: ${{ toJSON(needs) }} kubernetes-ha-e2e-result: name: Kubernetes HA E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, kubernetes-ha-e2e] + needs: [pr_metadata, kubernetes-ha-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' runs-on: ubuntu-latest steps: - - name: Verify Kubernetes HA E2E jobs - env: - BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }} - BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} - BUILD_CLI_RESULT: ${{ needs.build-cli.result }} - KUBERNETES_HA_E2E_RESULT: ${{ needs.kubernetes-ha-e2e.result }} - run: | - set -euo pipefail - failed=0 - for item in \ - "build-gateway:$BUILD_GATEWAY_RESULT" \ - "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ - "build-cli:$BUILD_CLI_RESULT" \ - "kubernetes-ha-e2e:$KUBERNETES_HA_E2E_RESULT"; do - name="${item%%:*}" - result="${item#*:}" - if [ "$result" != "success" ]; then - echo "::error::$name concluded $result" - failed=1 - fi - done - exit "$failed" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/check-job-results + with: + results: ${{ toJSON(needs) }} kubernetes-credential-drivers-e2e-result: name: Kubernetes Credential Drivers E2E result - needs: [pr_metadata, build-gateway, build-supervisor, kubernetes-credential-drivers-e2e] + needs: [pr_metadata, kubernetes-credential-drivers-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' runs-on: ubuntu-latest steps: - - name: Verify Kubernetes credential drivers E2E jobs - env: - BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }} - BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} - KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT: ${{ needs.kubernetes-credential-drivers-e2e.result }} - run: | - set -euo pipefail - failed=0 - for item in \ - "build-gateway:$BUILD_GATEWAY_RESULT" \ - "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ - "kubernetes-credential-drivers-e2e:$KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT"; do - name="${item%%:*}" - result="${item#*:}" - if [ "$result" != "success" ]; then - echo "::error::$name concluded $result" - failed=1 - fi - done - exit "$failed" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/check-job-results + with: + results: ${{ toJSON(needs) }} diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml new file mode 100644 index 0000000000..1305108fdf --- /dev/null +++ b/.github/workflows/build-binaries.yml @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Build Binary + +on: + workflow_call: + inputs: + package: + required: true + type: string + binary: + required: true + type: string + triple: + required: true + type: string + runner: + required: true + type: string + dev-shell: + required: true + type: string + cargo-version: + required: true + type: string + image-tag: + required: false + type: string + default: "" + artifact-name: + required: false + type: string + default: "" + extra-cargo-flags: + required: false + type: string + default: "" + interpreter: + required: false + type: string + default: "" + checkout-ref: + required: false + type: string + default: "" + secrets: + CACHIX_AUTH_TOKEN: + required: true + +permissions: + contents: read + +jobs: + build: + name: ${{ inputs.binary }} (${{ inputs.triple }}) + runs-on: ${{ inputs.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs['checkout-ref'] || github.sha }} + + - uses: ./.github/actions/setup-nix + with: + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - uses: ./.github/actions/build-rust-binary + with: + package: ${{ inputs.package }} + binary: ${{ inputs.binary }} + triple: ${{ inputs.triple }} + dev-shell: ${{ inputs['dev-shell'] }} + cargo-version: ${{ inputs['cargo-version'] }} + image-tag: ${{ inputs['image-tag'] }} + artifact-name: ${{ inputs['artifact-name'] }} + extra-cargo-flags: ${{ inputs['extra-cargo-flags'] }} + interpreter: ${{ inputs.interpreter }} diff --git a/.github/workflows/build-cli-binaries.yml b/.github/workflows/build-cli-binaries.yml new file mode 100644 index 0000000000..b8fa96dd7d --- /dev/null +++ b/.github/workflows/build-cli-binaries.yml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Build CLI Binaries + +on: + workflow_call: + inputs: + cargo-version: + required: true + type: string + image-tag: + required: false + type: string + default: "" + checkout-ref: + required: false + type: string + default: "" + secrets: + CACHIX_AUTH_TOKEN: + required: true + +permissions: + contents: read + +jobs: + build: + strategy: + matrix: + include: + - triple: x86_64-unknown-linux-musl + runner: linux-amd64-cpu8 + dev_shell: .#devShells.x86_64-linux.musl + - triple: aarch64-unknown-linux-musl + runner: linux-arm64-cpu8 + dev_shell: .#devShells.aarch64-linux.musl + - triple: aarch64-apple-darwin + runner: macos-15-xlarge + dev_shell: .#devShells.aarch64-darwin.default + uses: ./.github/workflows/build-binaries.yml + with: + package: openshell-cli + binary: openshell + triple: ${{ matrix.triple }} + runner: ${{ matrix.runner }} + dev-shell: ${{ matrix.dev_shell }} + cargo-version: ${{ inputs.cargo-version }} + image-tag: ${{ inputs.image-tag }} + checkout-ref: ${{ inputs.checkout-ref }} + secrets: inherit diff --git a/.github/workflows/build-gateway-binaries.yml b/.github/workflows/build-gateway-binaries.yml new file mode 100644 index 0000000000..ed5219e7c6 --- /dev/null +++ b/.github/workflows/build-gateway-binaries.yml @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Build Gateway Binaries + +on: + workflow_call: + inputs: + cargo-version: + required: true + type: string + image-tag: + required: false + type: string + default: "" + checkout-ref: + required: false + type: string + default: "" + secrets: + CACHIX_AUTH_TOKEN: + required: true + +permissions: + contents: read + +jobs: + build: + strategy: + matrix: + include: + - triple: x86_64-unknown-linux-gnu + runner: linux-amd64-cpu8 + dev_shell: .#devShells.x86_64-linux.glibc-2-28 + interpreter: /lib64/ld-linux-x86-64.so.2 + - triple: aarch64-unknown-linux-gnu + runner: linux-arm64-cpu8 + dev_shell: .#devShells.aarch64-linux.glibc-2-28 + interpreter: /lib/ld-linux-aarch64.so.1 + - triple: aarch64-apple-darwin + runner: macos-15-xlarge + dev_shell: .#devShells.aarch64-darwin.default + interpreter: "" + uses: ./.github/workflows/build-binaries.yml + with: + package: openshell-server + binary: openshell-gateway + triple: ${{ matrix.triple }} + runner: ${{ matrix.runner }} + dev-shell: ${{ matrix.dev_shell }} + cargo-version: ${{ inputs.cargo-version }} + image-tag: ${{ inputs.image-tag }} + interpreter: ${{ matrix.interpreter }} + checkout-ref: ${{ inputs.checkout-ref }} + secrets: inherit diff --git a/.github/workflows/build-sandbox-binaries.yml b/.github/workflows/build-sandbox-binaries.yml new file mode 100644 index 0000000000..53f81bec40 --- /dev/null +++ b/.github/workflows/build-sandbox-binaries.yml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Build Sandbox Binaries + +on: + workflow_call: + inputs: + cargo-version: + required: true + type: string + image-tag: + required: false + type: string + default: "" + checkout-ref: + required: false + type: string + default: "" + secrets: + CACHIX_AUTH_TOKEN: + required: true + +permissions: + contents: read + +jobs: + build: + strategy: + matrix: + include: + - triple: x86_64-unknown-linux-musl + runner: linux-amd64-cpu8 + dev_shell: .#devShells.x86_64-linux.musl + - triple: aarch64-unknown-linux-musl + runner: linux-arm64-cpu8 + dev_shell: .#devShells.aarch64-linux.musl + uses: ./.github/workflows/build-binaries.yml + with: + package: openshell-sandbox + binary: openshell-sandbox + triple: ${{ matrix.triple }} + runner: ${{ matrix.runner }} + dev-shell: ${{ matrix.dev_shell }} + cargo-version: ${{ inputs.cargo-version }} + image-tag: ${{ inputs.image-tag }} + checkout-ref: ${{ inputs.checkout-ref }} + secrets: inherit diff --git a/.github/workflows/build-vm-driver.yml b/.github/workflows/build-vm-driver.yml new file mode 100644 index 0000000000..a83c8c0834 --- /dev/null +++ b/.github/workflows/build-vm-driver.yml @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Build VM Driver + +on: + workflow_call: + inputs: + cargo-version: + required: true + type: string + image-tag: + required: false + type: string + default: "" + checkout-ref: + required: false + type: string + default: "" + secrets: + CACHIX_AUTH_TOKEN: + required: true + +permissions: + contents: read + +jobs: + build: + name: openshell-driver-vm (${{ matrix.triple }}) + strategy: + matrix: + include: + - arch: x86_64 + triple: x86_64-unknown-linux-gnu + runner: linux-amd64-cpu8 + dev_shell: .#devShells.x86_64-linux.glibc-2-28 + interpreter: /lib64/ld-linux-x86-64.so.2 + - arch: aarch64 + triple: aarch64-unknown-linux-gnu + runner: linux-arm64-cpu8 + dev_shell: .#devShells.aarch64-linux.glibc-2-28 + interpreter: /lib/ld-linux-aarch64.so.1 + - arch: aarch64 + triple: aarch64-apple-darwin + runner: macos-15-xlarge + dev_shell: .#devShells.aarch64-darwin.default + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + defaults: + run: + shell: nix develop ${{ matrix.dev_shell }} -c bash -euo pipefail {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs['checkout-ref'] || github.sha }} + + - uses: ./.github/actions/setup-nix + with: + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - name: Download openshell-sandbox + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: openshell-sandbox-${{ matrix.arch }}-unknown-linux-musl + path: sandbox + + - name: Build VM runtime + run: nix build .#vm-runtime + + - name: Compress Linux VM runtime + if: runner.os == 'Linux' + run: | + install -d target/vm-runtime-compressed + zstd -19 -T0 result/libkrun.so -o target/vm-runtime-compressed/libkrun.so.zst + zstd -19 -T0 result/libkrunfw.so.5 -o target/vm-runtime-compressed/libkrunfw.so.5.zst + zstd -19 -T0 result/gvproxy -o target/vm-runtime-compressed/gvproxy.zst + zstd -19 -T0 result/umoci -o target/vm-runtime-compressed/umoci.zst + + - name: Compress macOS VM runtime + if: runner.os == 'macOS' + run: | + install -d target/vm-runtime-compressed + zstd -19 -T0 result/libkrun.dylib -o target/vm-runtime-compressed/libkrun.dylib.zst + zstd -19 -T0 result/libkrunfw.5.dylib -o target/vm-runtime-compressed/libkrunfw.5.dylib.zst + zstd -19 -T0 result/gvproxy -o target/vm-runtime-compressed/gvproxy.zst + zstd -19 -T0 result/umoci -o target/vm-runtime-compressed/umoci.zst + + - name: Add openshell-sandbox to VM runtime + run: zstd -19 -T0 sandbox/openshell-sandbox -o target/vm-runtime-compressed/openshell-sandbox.zst + + - name: Build openshell-driver-vm + uses: ./.github/actions/build-rust-binary + env: + OPENSHELL_VM_RUNTIME_COMPRESSED_DIR: ${{ github.workspace }}/target/vm-runtime-compressed + with: + package: openshell-driver-vm + binary: openshell-driver-vm + triple: ${{ matrix.triple }} + dev-shell: ${{ matrix.dev_shell }} + cargo-version: ${{ inputs['cargo-version'] }} + image-tag: ${{ inputs['image-tag'] }} + interpreter: ${{ matrix.interpreter }} diff --git a/.github/workflows/deb-package.yml b/.github/workflows/deb-package.yml index 3929203909..7008acd4ed 100644 --- a/.github/workflows/deb-package.yml +++ b/.github/workflows/deb-package.yml @@ -49,28 +49,25 @@ jobs: - name: Download CLI artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: cli-linux-${{ matrix.arch }} - path: package-input/ + name: openshell-${{ matrix.cli_target }} + path: package-binaries/ - name: Download gateway artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: gateway-binary-linux-${{ matrix.arch }} - path: package-input/ + name: openshell-gateway-${{ matrix.gnu_target }} + path: package-binaries/ - name: Download VM driver artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: driver-vm-linux-${{ matrix.arch }} - path: package-input/ + name: openshell-driver-vm-${{ matrix.gnu_target }} + path: package-binaries/ - - name: Extract package inputs + - name: Configure package inputs run: | set -euo pipefail - mkdir -p package-binaries - tar -xzf "package-input/openshell-${{ matrix.cli_target }}.tar.gz" -C package-binaries - tar -xzf "package-input/openshell-gateway-${{ matrix.gnu_target }}.tar.gz" -C package-binaries - tar -xzf "package-input/openshell-driver-vm-${{ matrix.gnu_target }}.tar.gz" -C package-binaries + chmod +x package-binaries/openshell{,-gateway,-driver-vm} ls -lah package-binaries - name: Build Debian package @@ -79,10 +76,12 @@ jobs: OPENSHELL_CLI_BINARY="${PWD}/package-binaries/openshell" \ OPENSHELL_GATEWAY_BINARY="${PWD}/package-binaries/openshell-gateway" \ OPENSHELL_DRIVER_VM_BINARY="${PWD}/package-binaries/openshell-driver-vm" \ - OPENSHELL_DEB_VERSION="${{ inputs['deb-version'] }}" \ + OPENSHELL_DEB_VERSION="${INPUTS_DEB_VERSION}" \ OPENSHELL_DEB_ARCH="${{ matrix.deb_arch }}" \ OPENSHELL_OUTPUT_DIR=artifacts \ tasks/scripts/package-deb.sh + env: + INPUTS_DEB_VERSION: ${{ inputs['deb-version'] }} - name: Upload Debian package artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 73cb678e78..d27ba1e984 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,326 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + name: Docker Build on: workflow_call: inputs: component: - description: "Component to build (gateway, supervisor, cli)" required: true type: string - timeout-minutes: - description: "Per-arch Docker image job timeout in minutes" - required: false - type: number - default: 20 - push: - description: "Push image to registry" - required: false - type: boolean - default: true - platform: - description: "Target platform(s) for Docker build" - required: false - type: string - default: "linux/amd64,linux/arm64" - runner: - description: "Deprecated; per-arch native runners are selected automatically" - required: false + binary: + required: true type: string - default: "linux-amd64-cpu8" - cargo-version: - description: "Pre-computed cargo version (skips internal git-based computation)" - required: false + target-suffix: + required: true type: string - default: "" - auditable: - description: "Embed cargo-auditable dependency metadata in image binaries" - required: false - type: boolean - default: false image-tag: - description: "Image tag base to build/push (defaults to the GitHub SHA)" required: false type: string default: "" checkout-ref: - description: "Git ref to check out for build inputs (defaults to the workflow SHA)" required: false type: string default: "" - publish-manifest: - description: "Push the bare-SHA manifest. Set false for single-arch branch workflows." - required: false - type: boolean - default: true - -env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} permissions: contents: read packages: write -defaults: - run: - shell: bash - jobs: - resolve: - name: Resolve build plan - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.resolve.outputs.matrix }} - platform_count: ${{ steps.resolve.outputs.platform_count }} - arches: ${{ steps.resolve.outputs.arches }} - binary_component: ${{ steps.resolve.outputs.binary_component }} - binary_name: ${{ steps.resolve.outputs.binary_name }} - artifact_prefix: ${{ steps.resolve.outputs.artifact_prefix }} - image_tag_base: ${{ steps.resolve.outputs.image_tag_base }} - features: ${{ steps.resolve.outputs.features }} - has_image: ${{ steps.resolve.outputs.has_image }} - steps: - - name: Resolve component and platform matrix - id: resolve - run: | - set -euo pipefail - - component="${{ inputs.component }}" - case "$component" in - gateway) - binary_component=gateway - binary_name=openshell-gateway - features="bundled-z3" - has_image=true - ;; - supervisor) - binary_component=sandbox - binary_name=openshell-sandbox - features="" - has_image=true - ;; - cli) - binary_component=cli - binary_name=openshell - features="" - has_image=false - ;; - *) - echo "unsupported component: $component" >&2 - exit 1 - ;; - esac - - image_tag_base="${{ inputs['image-tag'] }}" - if [[ -z "$image_tag_base" ]]; then - image_tag_base="$GITHUB_SHA" - fi - - platform_input="${{ inputs.platform }}" - platform_input="${platform_input//[[:space:]]/}" - if [[ -z "$platform_input" ]]; then - echo "platform input must not be empty" >&2 - exit 1 - fi - - IFS=',' read -r -a platforms <<< "$platform_input" - matrix='{"include":[' - arches=() - count=0 - - for platform in "${platforms[@]}"; do - case "$platform" in - linux/amd64) - arch=amd64 - runner=linux-amd64-cpu8 - ;; - linux/arm64) - arch=arm64 - runner=linux-arm64-cpu8 - ;; - *) - echo "unsupported platform: $platform" >&2 - echo "supported platforms: linux/amd64, linux/arm64" >&2 - exit 1 - ;; - esac - - if [[ $count -gt 0 ]]; then - matrix+=',' - fi - matrix+='{"platform":"'"$platform"'","arch":"'"$arch"'","runner":"'"$runner"'"}' - arches+=("$arch") - count=$((count + 1)) - done - - matrix+=']}' - { - echo "matrix=$matrix" - echo "platform_count=$count" - echo "arches=${arches[*]}" - echo "binary_component=$binary_component" - echo "binary_name=$binary_name" - echo "artifact_prefix=rust-binary-${component}" - echo "image_tag_base=$image_tag_base" - echo "features=$features" - echo "has_image=$has_image" - } >> "$GITHUB_OUTPUT" - - rust-binary: - name: Rust ${{ needs.resolve.outputs.binary_component }} (${{ matrix.arch }}) - needs: resolve - permissions: - contents: read - packages: read - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.resolve.outputs.matrix) }} - uses: ./.github/workflows/rust-native-build.yml - with: - component: ${{ needs.resolve.outputs.binary_component }} - arch: ${{ matrix.arch }} - cargo-version: ${{ inputs['cargo-version'] }} - image-tag: ${{ needs.resolve.outputs.image_tag_base }} - checkout-ref: ${{ inputs['checkout-ref'] }} - features: ${{ needs.resolve.outputs.features }} - auditable: ${{ inputs.auditable }} - artifact-name: ${{ needs.resolve.outputs.artifact_prefix }}-linux-${{ matrix.arch }} - secrets: inherit - build: - name: Build ${{ inputs.component }} (${{ matrix.arch }}) - needs: [resolve, rust-binary] - if: needs.resolve.outputs.has_image == 'true' - runs-on: ${{ matrix.runner }} - timeout-minutes: ${{ inputs['timeout-minutes'] }} + name: ${{ inputs.component }} (${{ matrix.platform }}) strategy: fail-fast: false - matrix: ${{ fromJSON(needs.resolve.outputs.matrix) }} - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - # Expose the nv-gha-runners buildkitd.toml registry mirror config - # inside the container so setup-buildx can read it. - - /etc/buildkit:/etc/buildkit:ro - env: - IMAGE_TAG: ${{ format('{0}-{1}', needs.resolve.outputs.image_tag_base, matrix.arch) }} - IMAGE_REGISTRY: ghcr.io/nvidia/openshell - DOCKER_PUSH: ${{ inputs.push && '1' || '0' }} - DOCKER_PLATFORM: ${{ matrix.platform }} + matrix: + include: + - arch: amd64 + rust_arch: x86_64 + platform: linux/amd64 + runner: linux-amd64-cpu8 + - arch: arm64 + rust_arch: aarch64 + platform: linux/arm64 + runner: linux-arm64-cpu8 + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs['checkout-ref'] || github.sha }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Install tools - run: mise install --locked - - - name: Log in to GHCR - if: ${{ inputs.push }} - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up buildx (local driver) - uses: ./.github/actions/setup-buildx - with: - buildkitd-config: /etc/buildkit/buildkitd.toml - - name: Download Rust binary artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: ./.github/actions/build-docker-image with: - name: ${{ needs.resolve.outputs.artifact_prefix }}-linux-${{ matrix.arch }} - path: prebuilt-rust-binary - - - name: Stage Rust binary in Docker build context - run: | - set -euo pipefail - binary="${{ needs.resolve.outputs.binary_name }}" - download_dir="prebuilt-rust-binary" - stage="deploy/docker/.build/prebuilt-binaries/${{ matrix.arch }}" - found="$(find "$download_dir" -type f -name "$binary" -print -quit)" - if [[ -z "$found" ]]; then - echo "missing downloaded artifact file: $binary" >&2 - find "$download_dir" -maxdepth 4 -type f -print >&2 || true - exit 1 - fi - mkdir -p "$stage" - install -m 0755 "$found" "$stage/$binary" - ls -lh "$stage/" - - - name: Build ${{ inputs.component }} image - env: - DOCKER_BUILDER: openshell - run: | - set -euo pipefail - mise exec -- tasks/scripts/docker-build-image.sh "${{ inputs.component }}" \ - --cache-from "type=gha,scope=${{ inputs.component }}-${{ matrix.arch }}" \ - --cache-to "type=gha,mode=max,scope=${{ inputs.component }}-${{ matrix.arch }}" - - - name: Smoke check ${{ inputs.component }} image - run: | - set -euo pipefail - image="${IMAGE_REGISTRY}/${{ inputs.component }}:${IMAGE_TAG}" - case "${{ inputs.component }}" in - gateway) - output="$(docker run --rm --platform "${{ matrix.platform }}" "$image" --version)" - echo "$output" - grep -q '^openshell-gateway ' <<<"$output" - ;; - supervisor) - output="$(docker run --rm --platform "${{ matrix.platform }}" --entrypoint /openshell-sandbox "$image" --version)" - echo "$output" - grep -q '^openshell-sandbox ' <<<"$output" - ;; - esac - - merge: - name: Merge ${{ inputs.component }} manifest - needs: [resolve, build] - if: ${{ inputs.push && inputs['publish-manifest'] && needs.resolve.outputs.has_image == 'true' }} + component: ${{ inputs.component }} + binary: ${{ inputs.binary }} + triple: ${{ matrix.rust_arch }}-${{ inputs['target-suffix'] }} + arch: ${{ matrix.arch }} + platform: ${{ matrix.platform }} + image-tag: ${{ inputs.image-tag || github.sha }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + manifest: + name: ${{ inputs.component }} manifest + needs: build runs-on: linux-amd64-cpu8 timeout-minutes: 10 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - volumes: - - /var/run/docker.sock:/var/run/docker.sock steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs['checkout-ref'] || github.sha }} - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + shell: bash + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin - - name: Create multi-arch manifest + - name: Create manifest + shell: bash + env: + IMAGE_TAG: ${{ inputs.image-tag || github.sha }} + INPUTS_COMPONENT: ${{ inputs.component }} run: | - set -euo pipefail - image="ghcr.io/nvidia/openshell/${{ inputs.component }}" - refs=() - for arch in ${{ needs.resolve.outputs.arches }}; do - refs+=("${image}:${{ needs.resolve.outputs.image_tag_base }}-${arch}") - done + image=ghcr.io/nvidia/openshell/${INPUTS_COMPONENT} docker buildx imagetools create \ --prefer-index=false \ - -t "${image}:${{ needs.resolve.outputs.image_tag_base }}" \ - "${refs[@]}" + --tag "$image:${IMAGE_TAG}" \ + "$image:${IMAGE_TAG}-amd64" \ + "$image:${IMAGE_TAG}-arm64" - name: Verify merged manifest SBOM attestation + shell: bash env: - IMAGE_REF: ghcr.io/nvidia/openshell/${{ inputs.component }}:${{ needs.resolve.outputs.image_tag_base }} - REQUIRE_CARGO: ${{ inputs.auditable }} - run: | - args=("${IMAGE_REF}") - if [[ "${REQUIRE_CARGO}" == "true" ]]; then - args+=(--require-cargo) - fi - tasks/scripts/verify-image-sbom.sh "${args[@]}" + IMAGE_REF: ghcr.io/nvidia/openshell/${{ inputs.component }}:${{ inputs.image-tag || github.sha }} + run: tasks/scripts/verify-image-sbom.sh "${IMAGE_REF}" --require-cargo diff --git a/.github/workflows/driver-vm-linux.yml b/.github/workflows/driver-vm-linux.yml deleted file mode 100644 index 942cacdfbd..0000000000 --- a/.github/workflows/driver-vm-linux.yml +++ /dev/null @@ -1,221 +0,0 @@ -name: Driver VM Linux - -on: - workflow_call: - inputs: - cargo-version: - required: false - type: string - default: "" - image-tag: - required: true - type: string - checkout-ref: - required: true - type: string - runtime-platforms: - description: "Comma-separated VM runtime platforms to download: linux-aarch64,linux-x86_64" - required: false - type: string - default: "linux-aarch64,linux-x86_64" - driver-targets: - description: "JSON matrix entries for Linux VM driver targets" - required: false - type: string - default: >- - [{"arch":"arm64","runner":"linux-arm64-cpu8","target":"aarch64-unknown-linux-gnu","zig_target":"aarch64-unknown-linux-gnu.2.28","platform":"linux-aarch64","guest_arch":"aarch64"},{"arch":"amd64","runner":"linux-amd64-cpu8","target":"x86_64-unknown-linux-gnu","zig_target":"x86_64-unknown-linux-gnu.2.28","platform":"linux-x86_64","guest_arch":"x86_64"}] - -permissions: - contents: read - packages: read - -defaults: - run: - shell: bash - -jobs: - download-kernel-runtime: - name: Download Kernel Runtime - runs-on: linux-amd64-cpu8 - timeout-minutes: 10 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs['checkout-ref'] }} - - - name: Download Linux runtime tarballs - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RUNTIME_PLATFORMS: ${{ inputs['runtime-platforms'] }} - run: | - set -euo pipefail - mkdir -p runtime-artifacts - - runtime_platforms="${RUNTIME_PLATFORMS//[[:space:]]/}" - IFS=',' read -r -a platforms <<< "$runtime_platforms" - if [ "${#platforms[@]}" -eq 0 ]; then - echo "::error::No VM runtime platforms requested" - exit 1 - fi - for platform in "${platforms[@]}"; do - case "$platform" in - linux-aarch64 | linux-x86_64) ;; - *) - echo "::error::Unsupported VM runtime platform '$platform' in runtime-platforms" - exit 1 - ;; - esac - done - printf '%s\n' "${platforms[@]}" > runtime-artifacts/platforms.txt - - for platform in "${platforms[@]}"; do - asset="vm-runtime-${platform}.tar.zst" - echo "Downloading ${asset}..." - if ! gh release download vm-runtime \ - --repo "${GITHUB_REPOSITORY}" \ - --pattern "${asset}" \ - --dir runtime-artifacts \ - --clobber; then - echo "::error::No ${asset} asset found on vm-runtime release" - exit 1 - fi - done - - ls -lah runtime-artifacts/ - - - name: Verify downloads - run: | - set -euo pipefail - while IFS= read -r platform; do - test -f "runtime-artifacts/vm-runtime-${platform}.tar.zst" - done < runtime-artifacts/platforms.txt - - - name: Upload runtime artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: vm-driver-kernel-runtime-tarballs - path: runtime-artifacts/vm-runtime-*.tar.zst - retention-days: 1 - - build-driver-vm-linux: - name: Build Driver VM (Linux ${{ matrix.arch }}) - needs: [download-kernel-runtime] - strategy: - matrix: - include: ${{ fromJSON(inputs['driver-targets']) }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs['checkout-ref'] }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: driver-vm-linux-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* - - - name: Download kernel runtime tarball - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: vm-driver-kernel-runtime-tarballs - path: runtime-download/ - - - name: Stage compressed runtime for embedding - run: | - set -euo pipefail - COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" - VM_RUNTIME_TARBALL="${PWD}/runtime-download/vm-runtime-${{ matrix.platform }}.tar.zst" \ - VM_RUNTIME_PLATFORM="${{ matrix.platform }}" \ - OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="$COMPRESSED_DIR" \ - tasks/scripts/vm/compress-vm-runtime.sh - - - name: Build bundled supervisor - run: | - set -euo pipefail - OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" \ - tasks/scripts/vm/build-supervisor-bundle.sh --arch "${{ matrix.guest_arch }}" - - - name: Verify embedded driver inputs - run: | - set -euo pipefail - for file in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do - test -s "target/vm-runtime-compressed/${file}" - done - - - name: Scope workspace to driver-vm crates - run: | - set -euo pipefail - sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml - - - name: Patch workspace version - if: ${{ inputs['cargo-version'] != '' }} - run: | - set -euo pipefail - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ inputs['cargo-version'] }}"'"/}' Cargo.toml - - - name: Build openshell-driver-vm with glibc 2.28 floor - run: | - set -euo pipefail - mise x -- rustup target add ${{ matrix.target }} - OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" \ - mise x -- cargo zigbuild --release --target ${{ matrix.zig_target }} -p openshell-driver-vm --bin openshell-driver-vm - mkdir -p artifacts/bin - install -m 0755 target/${{ matrix.target }}/release/openshell-driver-vm artifacts/bin/openshell-driver-vm - - - name: Verify packaged binary - run: | - set -euo pipefail - OUTPUT="$(artifacts/bin/openshell-driver-vm --version)" - echo "$OUTPUT" - grep -q '^openshell-driver-vm ' <<<"$OUTPUT" - - - name: Verify glibc symbol floor - run: tasks/scripts/verify-glibc-symbols.sh 2.28 artifacts/bin/openshell-driver-vm - - - name: sccache stats - if: always() - run: mise x -- sccache --show-stats - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf "artifacts/openshell-driver-vm-${{ matrix.target }}.tar.gz" \ - -C artifacts/bin openshell-driver-vm - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: driver-vm-linux-${{ matrix.arch }} - path: artifacts/*.tar.gz - retention-days: 5 diff --git a/.github/workflows/driver-vm-macos.yml b/.github/workflows/driver-vm-macos.yml deleted file mode 100644 index a97ade9cbb..0000000000 --- a/.github/workflows/driver-vm-macos.yml +++ /dev/null @@ -1,221 +0,0 @@ -name: Driver VM macOS - -on: - workflow_call: - inputs: - cargo-version: - required: true - type: string - image-tag: - required: true - type: string - checkout-ref: - required: true - type: string - -permissions: - contents: read - packages: read - -defaults: - run: - shell: bash - -jobs: - download-kernel-runtime: - name: Download Kernel Runtime - runs-on: linux-amd64-cpu8 - timeout-minutes: 10 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs['checkout-ref'] }} - - - name: Download macOS runtime tarball - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - mkdir -p runtime-artifacts - - asset="vm-runtime-darwin-aarch64.tar.zst" - echo "Downloading ${asset}..." - if ! gh release download vm-runtime \ - --repo "${GITHUB_REPOSITORY}" \ - --pattern "${asset}" \ - --dir runtime-artifacts \ - --clobber; then - echo "::error::No ${asset} asset found on vm-runtime release" - exit 1 - fi - - ls -lah runtime-artifacts/ - - - name: Verify download - run: test -f runtime-artifacts/vm-runtime-darwin-aarch64.tar.zst - - - name: Upload runtime artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: vm-driver-macos-kernel-runtime-tarball - path: runtime-artifacts/vm-runtime-darwin-aarch64.tar.zst - retention-days: 1 - - build-supervisor-arm64: - name: Build Supervisor Bundle (arm64) - runs-on: linux-arm64-cpu8 - timeout-minutes: 30 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs['checkout-ref'] }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: driver-vm-supervisor-arm64 - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* - - - name: Build bundled supervisor - run: | - set -euo pipefail - tasks/scripts/vm/build-supervisor-bundle.sh --arch aarch64 - - - name: sccache stats - if: always() - run: mise x -- sccache --show-stats - - - name: Upload supervisor bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: driver-vm-supervisor-arm64 - path: target/vm-runtime-compressed/openshell-sandbox.zst - retention-days: 1 - - build-driver-vm-macos: - name: Build Driver VM (macOS) - needs: [download-kernel-runtime, build-supervisor-arm64] - runs-on: linux-amd64-cpu8 - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs['checkout-ref'] }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* - - - name: Download kernel runtime tarball - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: vm-driver-macos-kernel-runtime-tarball - path: runtime-download/ - - - name: Prepare compressed runtime directory - run: | - set -euo pipefail - COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed-macos" - VM_RUNTIME_TARBALL="${PWD}/runtime-download/vm-runtime-darwin-aarch64.tar.zst" \ - VM_RUNTIME_PLATFORM="darwin-aarch64" \ - OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="$COMPRESSED_DIR" \ - tasks/scripts/vm/compress-vm-runtime.sh - - - name: Download bundled supervisor - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: driver-vm-supervisor-arm64 - path: target/vm-runtime-compressed-macos/ - - - name: Verify bundled supervisor - run: | - set -euo pipefail - test -f target/vm-runtime-compressed-macos/openshell-sandbox.zst - ls -lh target/vm-runtime-compressed-macos/openshell-sandbox.zst - - - name: Verify embedded driver inputs - run: | - set -euo pipefail - for file in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do - test -s "target/vm-runtime-compressed-macos/${file}" - done - - - name: Build macOS binary via Docker - run: | - set -euo pipefail - docker buildx build \ - --file deploy/docker/Dockerfile.driver-vm-macos \ - --build-arg OPENSHELL_CARGO_VERSION="${{ inputs['cargo-version'] }}" \ - --build-arg OPENSHELL_IMAGE_TAG="${{ inputs['image-tag'] }}" \ - --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ - --build-context vm-runtime-compressed="${PWD}/target/vm-runtime-compressed-macos" \ - --target binary \ - --output type=local,dest=out/ \ - . - - - name: Verify packaged binary shape - run: test -x out/openshell-driver-vm - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-driver-vm-aarch64-apple-darwin.tar.gz \ - -C out openshell-driver-vm - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: driver-vm-macos - path: artifacts/*.tar.gz - retention-days: 5 diff --git a/.github/workflows/e2e-docker-test.yml b/.github/workflows/e2e-docker-test.yml new file mode 100644 index 0000000000..aa884bc3e2 --- /dev/null +++ b/.github/workflows/e2e-docker-test.yml @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Docker E2E Test + +on: + workflow_call: + inputs: + image-tag: + required: true + type: string + runner: + required: false + type: string + default: linux-amd64-cpu8 + checkout-ref: + required: false + type: string + default: "" + gateway-artifact: + required: false + type: string + default: "" + external-driver-binary: + required: false + type: string + default: "" + suite-matrix: + required: false + type: string + default: >- + [{"suite":"python","cmd":"mise run --no-deps --skip-deps e2e:python","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-python","cmd":"mise run --no-deps --skip-deps e2e:oidc-python:docker","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-pkce-docker","cmd":"mise run --no-deps --skip-deps e2e:oidc-pkce:docker","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"rust-docker","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"mcp","cmd":"mise run --no-deps --skip-deps e2e:mcp","apt_packages":"","python_proto":false,"mcp":true}] + +permissions: + actions: read + contents: read + packages: read + +jobs: + e2e: + name: E2E (${{ matrix.suite }}) + runs-on: ${{ inputs.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.suite-matrix) }} + container: + image: ghcr.io/nvidia/openshell/ci:37072ee81cd7b294c714bfa5ecc829b6927b3d70 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /home/runner/_work:/home/runner/_work + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAG: ${{ inputs.image-tag }} + OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell + OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-ref || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/setup-e2e-cli + + - uses: ./.github/actions/setup-e2e-gateway + with: + artifact-name: ${{ inputs.gateway-artifact }} + + - if: inputs.external-driver-binary != '' + uses: ./.github/actions/setup-e2e-driver + with: + binary: ${{ inputs.external-driver-binary }} + + - name: Check out MCP conformance tests + if: matrix.mcp + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: modelcontextprotocol/conformance + ref: b9041ea41b0188581803459dbae71bc7e02fd995 + path: .cache/mcp-conformance + persist-credentials: false + + - name: Install OS test dependencies + if: matrix.apt_packages != '' + shell: bash + env: + APT_PACKAGES: ${{ matrix.apt_packages }} + run: | + read -ra apt_packages <<< "$APT_PACKAGES" + apt-get update + apt-get install -y "${apt_packages[@]}" + rm -rf /var/lib/apt/lists/* + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + + - name: Install Python dependencies and generate protobuf stubs + if: matrix.python_proto + run: uv sync --frozen && mise run --no-deps python:proto + + - name: Run tests + env: + OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE: ${{ format('openshell-mcp-conformance-client:{0}', inputs.image-tag) }} + run: ${{ matrix.cmd }} diff --git a/.github/workflows/e2e-gpu-test.yaml b/.github/workflows/e2e-gpu-test.yaml index be2b66e1d4..1a8275a274 100644 --- a/.github/workflows/e2e-gpu-test.yaml +++ b/.github/workflows/e2e-gpu-test.yaml @@ -7,16 +7,6 @@ on: description: "Image tag to test (typically the commit SHA)" required: true type: string - cli-artifact-prefix: - description: "Optional prebuilt CLI artifact prefix (artifact suffix is linux-)" - required: false - type: string - default: "" - gateway-artifact-prefix: - description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-)" - required: false - type: string - default: "" permissions: actions: read @@ -70,16 +60,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Use prebuilt OpenShell CLI - if: inputs.cli-artifact-prefix != '' uses: ./.github/actions/setup-e2e-cli - with: - artifact-prefix: ${{ inputs.cli-artifact-prefix }} - name: Use prebuilt OpenShell gateway - if: inputs.gateway-artifact-prefix != '' uses: ./.github/actions/setup-e2e-gateway - with: - artifact-prefix: ${{ inputs.gateway-artifact-prefix }} - name: Log in to GHCR run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index 3e36570144..4498f9d44a 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -47,11 +47,21 @@ on: required: false type: string default: "v2026.4.25" - cli-artifact-prefix: - description: "Optional prebuilt CLI artifact prefix (artifact suffix is linux-)" + gateway-artifact: + description: "Optional prebuilt gateway artifact used to compose a local test image" required: false type: string default: "" + external-driver-binary: + description: "Optional standalone compute driver binary used to compose a local test image" + required: false + type: string + default: "" + cluster-images: + description: "Space-separated published images to preload into kind" + required: false + type: string + default: "gateway supervisor" permissions: actions: read @@ -81,62 +91,29 @@ jobs: ref: ${{ inputs['checkout-ref'] || github.sha }} - name: Use prebuilt OpenShell CLI - if: inputs.cli-artifact-prefix != '' uses: ./.github/actions/setup-e2e-cli - with: - artifact-prefix: ${{ inputs.cli-artifact-prefix }} - - name: Install mise - run: | - curl https://mise.run | MISE_VERSION=v2026.4.25 sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" - - - name: Install tools - run: mise install --locked - - # The openshell-policy crate transitively pulls in z3-sys, whose - # build script needs the z3 C/C++ headers, clang/bindgen, and CMake to - # compile both system-linked and bundled-Z3 builds. The bare runner - # doesn't ship them; the CI container - # image used by other Rust e2e jobs does, but we can't run this job - # there (the runner's container handler injects its own --network - # bridge, which conflicts with the --network host we need so kind's - # API server is reachable from the test process). - - name: Install z3 build deps - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends libz3-dev clang cmake - - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Create kind cluster - uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 + - name: Use prebuilt OpenShell gateway + if: inputs.gateway-artifact != '' + uses: ./.github/actions/setup-e2e-gateway with: - cluster_name: ${{ env.KIND_CLUSTER_NAME }} - wait: 120s + artifact-name: ${{ inputs.gateway-artifact }} - # mise.toml sets KUBECONFIG="{{config_root}}/kubeconfig"; helm/kind-action - # writes to ~/.kube/config. Materialize the kind context at the mise path - # so `mise run e2e:kubernetes` (and the wrapper's `kubectl --context=...`) - # finds the kind cluster. - - name: Export kind kubeconfig to mise path - run: | - set -euo pipefail - kind get kubeconfig --name "$KIND_CLUSTER_NAME" > "$GITHUB_WORKSPACE/kubeconfig" - chmod 600 "$GITHUB_WORKSPACE/kubeconfig" + - name: Use prebuilt external compute driver + if: inputs.external-driver-binary != '' + uses: ./.github/actions/setup-e2e-driver + with: + binary: ${{ inputs.external-driver-binary }} - - name: Load gateway and supervisor images into kind - run: | - set -euo pipefail - for component in gateway supervisor; do - image="ghcr.io/nvidia/openshell/${component}:${{ inputs.image-tag }}" - archive="${RUNNER_TEMP:-/tmp}/openshell-${component}-linux-amd64.tar" - docker pull --platform linux/amd64 "$image" - docker image save --platform linux/amd64 --output "$archive" "$image" - kind load image-archive "$archive" --name "$KIND_CLUSTER_NAME" - done + - name: Set up kind cluster + uses: ./.github/actions/setup-e2e-kind + with: + cluster-name: ${{ env.KIND_CLUSTER_NAME }} + image-tag: ${{ inputs.image-tag }} + images: ${{ inputs.cluster-images }} + registry-username: ${{ github.actor }} + registry-password: ${{ secrets.GITHUB_TOKEN }} + mise-version: ${{ inputs.mise-version }} - name: Run Kubernetes E2E env: diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml new file mode 100644 index 0000000000..4a5a6d38d7 --- /dev/null +++ b/.github/workflows/e2e-podman-test.yml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Podman E2E Test + +on: + workflow_call: + inputs: + image-tag: + required: true + type: string + checkout-ref: + required: false + type: string + default: "" + gateway-artifact: + required: false + type: string + default: "" + external-driver-binary: + required: false + type: string + default: "" + suite-matrix: + required: false + type: string + default: >- + [{"suite":"provider-refresh-keycloak","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:provider-refresh-keycloak"}] + +permissions: + actions: read + contents: read + packages: read + +jobs: + e2e: + name: E2E (rust-podman-${{ matrix.suite }}, ${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.suite-matrix) }} + env: + IMAGE_TAG: ${{ inputs.image-tag }} + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell + OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-ref || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/setup-e2e-cli + + - uses: ./.github/actions/setup-e2e-gateway + with: + artifact-name: ${{ inputs.gateway-artifact }} + + - if: inputs.external-driver-binary != '' + uses: ./.github/actions/setup-e2e-driver + with: + binary: ${{ inputs.external-driver-binary }} + + - uses: ./.github/actions/setup-e2e-podman + with: + podman-major: ${{ matrix.podman_major }} + podman-package-version: ${{ matrix.podman_package_version }} + conmon-package-version: ${{ matrix.conmon_package_version }} + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + + - name: Run tests + run: ${{ matrix.cmd }} + + - name: Print AppArmor denials + if: always() + run: sudo dmesg | grep -E 'apparmor=.*DENIED|profile="unprivileged_userns"' | tail -100 || true + + - name: Fail on pasta SIGTERM AppArmor denial + if: always() + run: | + set -euo pipefail + denials="$(sudo dmesg | grep -E 'profile="pasta".*requested_mask="receive".*signal=term.*peer="podman"' || true)" + if [ -n "${denials}" ]; then + echo "::error::pasta denied Podman's SIGTERM; Podman will use its SIGKILL fallback" + printf '%s\n' "${denials}" + exit 1 + fi diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml deleted file mode 100644 index d3e4230636..0000000000 --- a/.github/workflows/e2e-test.yml +++ /dev/null @@ -1,234 +0,0 @@ -name: E2E Test - -on: - workflow_call: - inputs: - image-tag: - description: "Image tag to test (typically the commit SHA)" - required: true - type: string - runner: - description: "GitHub Actions runner label" - required: false - type: string - default: "linux-amd64-cpu8" - checkout-ref: - description: "Git ref to check out for test inputs (defaults to the workflow SHA)" - required: false - type: string - default: "" - cli-artifact-prefix: - description: "Optional prebuilt CLI artifact prefix (artifact suffix is linux-)" - required: false - type: string - default: "" - gateway-artifact-prefix: - description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-)" - required: false - type: string - default: "" - vm-driver-artifact-name: - description: "Optional prebuilt VM driver artifact name" - required: false - type: string - default: "" - -permissions: - actions: read - contents: read - packages: read - -jobs: - e2e: - name: "E2E (${{ matrix.suite }})" - runs-on: ${{ inputs.runner }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - suite: python - cmd: "mise run --no-deps --skip-deps e2e:python" - apt_packages: "" - - suite: oidc-python - cmd: "mise run --no-deps --skip-deps e2e:oidc-python:docker" - apt_packages: "" - - suite: oidc-pkce-docker - cmd: "mise run --no-deps --skip-deps e2e:oidc-pkce:docker" - apt_packages: "openssh-client" - - suite: rust-docker - cmd: "mise run --no-deps --skip-deps e2e:rust" - apt_packages: "openssh-client" - - suite: rust-docker-external-driver - cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:docker:external-driver" - apt_packages: "openssh-client" - - suite: mcp - cmd: "mise run --no-deps --skip-deps e2e:mcp" - apt_packages: "" - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - /home/runner/_work:/home/runner/_work - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IMAGE_TAG: ${{ inputs.image-tag }} - OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell - OPENSHELL_REGISTRY_HOST: ghcr.io - OPENSHELL_REGISTRY_NAMESPACE: nvidia/openshell - OPENSHELL_REGISTRY_USERNAME: ${{ github.actor }} - OPENSHELL_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs['checkout-ref'] || github.sha }} - persist-credentials: false - - - name: Use prebuilt OpenShell CLI - if: inputs.cli-artifact-prefix != '' - uses: ./.github/actions/setup-e2e-cli - with: - artifact-prefix: ${{ inputs.cli-artifact-prefix }} - - - name: Use prebuilt OpenShell gateway - if: inputs.gateway-artifact-prefix != '' - uses: ./.github/actions/setup-e2e-gateway - with: - artifact-prefix: ${{ inputs.gateway-artifact-prefix }} - - - name: Check out MCP conformance tests - if: matrix.suite == 'mcp' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: modelcontextprotocol/conformance - # Pin after v0.1.16 to include the tools_call client scenario fix. - ref: b9041ea41b0188581803459dbae71bc7e02fd995 - path: .cache/mcp-conformance - persist-credentials: false - - - name: Install OS test dependencies - if: matrix.apt_packages != '' - env: - APT_PACKAGES: ${{ matrix.apt_packages }} - run: apt-get update && apt-get install -y ${APT_PACKAGES} && rm -rf /var/lib/apt/lists/* - - - name: Log in to GHCR with Docker - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Install Python dependencies and generate protobuf stubs - if: matrix.suite == 'python' || matrix.suite == 'oidc-python' - run: uv sync --frozen && mise run --no-deps python:proto - - - name: Run tests - env: - OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }} - OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE: ${{ format('openshell-mcp-conformance-client:{0}', inputs.image-tag) }} - run: ${{ matrix.cmd }} - - e2e-vm: - name: E2E (rust-vm-${{ matrix.suite }}) - # libkrun needs KVM, so this job must run directly on a GitHub-hosted - # Linux VM. GitHub-hosted macOS runners do not support nested - # virtualization, and a job container would hide the host KVM device. - runs-on: ubuntu-24.04 - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - include: - - suite: managed - cmd: "mise run --no-deps --skip-deps e2e:vm" - - suite: external-driver - cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:vm:external-driver" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs['checkout-ref'] || github.sha }} - persist-credentials: false - - - name: Use prebuilt OpenShell CLI - if: inputs.cli-artifact-prefix != '' - uses: ./.github/actions/setup-e2e-cli - with: - artifact-prefix: ${{ inputs.cli-artifact-prefix }} - - - name: Use prebuilt OpenShell gateway - if: inputs.gateway-artifact-prefix != '' - uses: ./.github/actions/setup-e2e-gateway - with: - artifact-prefix: ${{ inputs.gateway-artifact-prefix }} - - - name: Use prebuilt OpenShell VM driver - if: inputs.vm-driver-artifact-name != '' - uses: ./.github/actions/setup-e2e-vm-driver - with: - artifact-name: ${{ inputs.vm-driver-artifact-name }} - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - build-essential \ - clang \ - cmake \ - libclang-dev \ - libssl-dev \ - libz3-dev \ - openssh-client \ - pkg-config \ - socat \ - zstd - - - name: Enable KVM access - run: | - set -euo pipefail - if [[ ! -c /dev/kvm ]]; then - echo "::error::The GitHub-hosted runner did not expose /dev/kvm" - lscpu - grep -m1 -E '^(flags|Features)' /proc/cpuinfo || true - ls -la /dev - exit 1 - fi - - # Package installation can restart systemd-udevd, which reapplies - # the default root:kvm 0660 mode. Install a persistent rule after - # dependencies so later udev events preserve runner access. - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --settle --name-match=kvm - - ls -l /dev/kvm - exec 3<>/dev/kvm - exec 3>&- - - - name: Validate VM host tools - run: | - command -v mke2fs - command -v mkfs.ext4 - command -v debugfs - - - name: Install mise - run: | - curl https://mise.run | MISE_VERSION=v2026.4.25 sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: e2e-vm-linux-amd64 - cache-on-failure: "true" - - - name: Run VM E2E - run: ${{ matrix.cmd }} diff --git a/.github/workflows/e2e-vm-test.yml b/.github/workflows/e2e-vm-test.yml new file mode 100644 index 0000000000..57e04c32b3 --- /dev/null +++ b/.github/workflows/e2e-vm-test.yml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: VM E2E Test + +on: + workflow_call: + inputs: + checkout-ref: + required: false + type: string + default: "" + gateway-artifact: + required: false + type: string + default: "" + suite-name: + required: false + type: string + default: managed + e2e-task: + required: false + type: string + default: e2e:vm + +permissions: + actions: read + contents: read + packages: read + +jobs: + e2e: + name: E2E (rust-vm-${{ inputs.suite-name }}) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-ref || github.sha }} + persist-credentials: false + + - uses: ./.github/actions/setup-e2e-cli + + - uses: ./.github/actions/setup-e2e-gateway + with: + artifact-name: ${{ inputs.gateway-artifact }} + + - uses: ./.github/actions/setup-e2e-vm-driver + + - uses: ./.github/actions/setup-e2e-vm + + - name: Run tests + env: + E2E_TASK: ${{ inputs.e2e-task }} + run: mise run --no-deps --skip-deps "$E2E_TASK" diff --git a/.github/workflows/package-release-binaries.yml b/.github/workflows/package-release-binaries.yml new file mode 100644 index 0000000000..f1900f3c35 --- /dev/null +++ b/.github/workflows/package-release-binaries.yml @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Package Release Binaries + +on: + workflow_call: + +permissions: + actions: read + contents: read + +jobs: + package: + name: Package ${{ matrix.artifact }} + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - artifact: openshell-x86_64-unknown-linux-musl + binary: openshell + package: cli-linux-amd64 + - artifact: openshell-aarch64-unknown-linux-musl + binary: openshell + package: cli-linux-arm64 + - artifact: openshell-aarch64-apple-darwin + binary: openshell + package: cli-macos + - artifact: openshell-gateway-x86_64-unknown-linux-gnu + binary: openshell-gateway + package: gateway-binary-linux-amd64 + - artifact: openshell-gateway-aarch64-unknown-linux-gnu + binary: openshell-gateway + package: gateway-binary-linux-arm64 + - artifact: openshell-gateway-aarch64-apple-darwin + binary: openshell-gateway + package: gateway-binary-macos + - artifact: openshell-sandbox-x86_64-unknown-linux-musl + binary: openshell-sandbox + package: supervisor-binary-linux-amd64 + - artifact: openshell-sandbox-aarch64-unknown-linux-musl + binary: openshell-sandbox + package: supervisor-binary-linux-arm64 + - artifact: openshell-driver-vm-x86_64-unknown-linux-gnu + binary: openshell-driver-vm + package: driver-vm-linux-amd64 + - artifact: openshell-driver-vm-aarch64-unknown-linux-gnu + binary: openshell-driver-vm + package: driver-vm-linux-arm64 + - artifact: openshell-driver-vm-aarch64-apple-darwin + binary: openshell-driver-vm + package: driver-vm-macos + steps: + - name: Download ${{ matrix.artifact }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ matrix.artifact }} + path: raw + + - name: Create archive + env: + ARTIFACT: ${{ matrix.artifact }} + BINARY: ${{ matrix.binary }} + run: | + set -euo pipefail + chmod +x "raw/${BINARY}" + tar -czf "${ARTIFACT}.tar.gz" -C raw "${BINARY}" + + - name: Upload ${{ matrix.package }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ matrix.package }} + path: ${{ matrix.artifact }}.tar.gz + retention-days: 5 + if-no-files-found: error diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 500fcb4be6..7982a81643 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -53,41 +53,114 @@ jobs: echo "rpm_version=$(uv run python tasks/scripts/release.py get-version --rpm-version)" >> "$GITHUB_OUTPUT" echo "rpm_release=$(uv run python tasks/scripts/release.py get-version --rpm-release)" >> "$GITHUB_OUTPUT" + build-cli: + needs: compute-versions + permissions: + contents: read + uses: ./.github/workflows/build-cli-binaries.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: dev + secrets: inherit + build-gateway: - needs: [compute-versions] + needs: compute-versions + permissions: + contents: read + uses: ./.github/workflows/build-gateway-binaries.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: dev + secrets: inherit + + build-sandbox: + needs: compute-versions + permissions: + contents: read + uses: ./.github/workflows/build-sandbox-binaries.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: dev + secrets: inherit + + build-vm-driver: + needs: [compute-versions, build-sandbox] + permissions: + contents: read + uses: ./.github/workflows/build-vm-driver.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: dev + secrets: inherit + + package-binaries: + needs: [build-cli, build-gateway, build-sandbox, build-vm-driver] + permissions: + actions: read + contents: read + uses: ./.github/workflows/package-release-binaries.yml + + build-gateway-image: + needs: build-gateway + permissions: + contents: read + packages: write uses: ./.github/workflows/docker-build.yml with: component: gateway - cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} - auditable: true + binary: openshell-gateway + target-suffix: unknown-linux-gnu + secrets: inherit - build-supervisor: - needs: [compute-versions] + build-supervisor-image: + needs: build-sandbox + permissions: + contents: read + packages: write uses: ./.github/workflows/docker-build.yml with: component: supervisor - cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} - auditable: true + binary: openshell-sandbox + target-suffix: unknown-linux-musl + secrets: inherit - e2e: - needs: [build-gateway, build-supervisor] + docker-e2e: + needs: [build-cli, build-gateway, build-supervisor-image] permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-test.yml + uses: ./.github/workflows/e2e-docker-test.yml with: image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 + podman-e2e: + needs: [build-cli, build-gateway, build-supervisor-image] + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-podman-test.yml + with: + image-tag: ${{ github.sha }} + + vm-e2e: + needs: [build-cli, build-gateway, build-vm-driver] + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-vm-test.yml + tag-ghcr-dev: name: Tag GHCR Images as Dev - needs: [build-gateway, build-supervisor, release-dev] + needs: [build-gateway-image, build-supervisor-image, release-dev] runs-on: linux-amd64-cpu8 timeout-minutes: 10 steps: - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin - name: Tag images as dev run: | @@ -138,456 +211,10 @@ jobs: path: target/wheels/*.whl retention-days: 5 - # --------------------------------------------------------------------------- - # Build CLI binaries (Linux musl — static, native on each arch) - # - # Builds run directly on the CI host (glibc Ubuntu). Zig provides musl - # C/C++ toolchains for bundled-z3 and ring, and is also used as the linker. - # --------------------------------------------------------------------------- - build-cli-linux: - name: Build CLI (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - target: x86_64-unknown-linux-musl - zig_target: x86_64-linux-musl - - arch: arm64 - runner: linux-arm64-cpu8 - target: aarch64-unknown-linux-musl - zig_target: aarch64-linux-musl - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: dev - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: cli-musl-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Add Rust musl target - run: mise x -- rustup target add ${{ matrix.target }} - - - name: Set up zig musl wrappers - run: | - set -euo pipefail - ZIG="$(mise which zig)" - ZIG_TARGET="${{ matrix.zig_target }}" - mkdir -p /tmp/zig-musl - - # cc-rs injects --target= (for example - # aarch64-unknown-linux-musl), which zig does not parse. Strip any - # caller-provided --target and use the wrapper's zig-native target. - for tool in cc c++; do - printf '#!/bin/bash\nargs=()\nfor arg in "$@"; do\n case "$arg" in\n --target=*) ;;\n *) args+=("$arg") ;;\n esac\ndone\nexec "%s" %s --target=%s "${args[@]}"\n' \ - "$ZIG" "$tool" "$ZIG_TARGET" > "/tmp/zig-musl/${tool}" - chmod +x "/tmp/zig-musl/${tool}" - done - - TARGET_ENV=$(echo "${{ matrix.target }}" | tr '-' '_') - TARGET_ENV_UPPER=${TARGET_ENV^^} - - # Use zig for C/C++ compilation and final linking. - echo "CC_${TARGET_ENV}=/tmp/zig-musl/cc" >> "$GITHUB_ENV" - echo "CXX_${TARGET_ENV}=/tmp/zig-musl/c++" >> "$GITHUB_ENV" - echo "CARGO_TARGET_${TARGET_ENV_UPPER}_LINKER=/tmp/zig-musl/cc" >> "$GITHUB_ENV" - - # Let zig own CRT/startfiles to avoid duplicate _start symbols. - echo "CARGO_TARGET_${TARGET_ENV_UPPER}_RUSTFLAGS=-Clink-self-contained=no" >> "$GITHUB_ENV" - - # z3 built with zig c++ uses libc++ symbols (std::__1::*). - # Override z3-sys default (stdc++) so Rust links the matching runtime. - echo "CXXSTDLIB=c++" >> "$GITHUB_ENV" - - - name: Patch workspace version - if: needs.compute-versions.outputs.cargo_version != '' - run: | - set -euo pipefail - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml - - - name: Build ${{ matrix.target }} - run: mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-cli - - - name: sccache stats - if: always() - run: mise x -- sccache --show-stats - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-${{ matrix.target }}.tar.gz \ - -C target/${{ matrix.target }}/release openshell - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: cli-linux-${{ matrix.arch }} - path: artifacts/*.tar.gz - retention-days: 5 - - # --------------------------------------------------------------------------- - # Build CLI binary (macOS aarch64 via osxcross) - # --------------------------------------------------------------------------- - build-cli-macos: - name: Build CLI (macOS) - needs: [compute-versions] - runs-on: linux-amd64-cpu8 - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - - name: Build macOS binary via Docker - run: | - set -euo pipefail - docker buildx build \ - --file deploy/docker/Dockerfile.cli-macos \ - --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ - --build-arg OPENSHELL_IMAGE_TAG=dev \ - --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ - --target binary \ - --output type=local,dest=out/ \ - . - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-aarch64-apple-darwin.tar.gz \ - -C out openshell - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: cli-macos - path: artifacts/*.tar.gz - retention-days: 5 - - # --------------------------------------------------------------------------- - # Build standalone gateway binaries (Linux GNU — glibc 2.28 floor) - # --------------------------------------------------------------------------- - build-gateway-binary-linux: - name: Build Gateway Binary (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - target: x86_64-unknown-linux-gnu - zig_target: x86_64-unknown-linux-gnu.2.28 - - arch: arm64 - runner: linux-arm64-cpu8 - target: aarch64-unknown-linux-gnu - zig_target: aarch64-unknown-linux-gnu.2.28 - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: gateway-binary-gnu-${{ matrix.arch }}-zig-wrapper-${{ hashFiles('tasks/scripts/setup-zig-cc-wrapper.sh') }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Patch workspace version - if: needs.compute-versions.outputs.cargo_version != '' - run: | - set -euo pipefail - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml - - - name: Set up Zig C/C++ wrappers - run: tasks/scripts/setup-zig-cc-wrapper.sh ${{ matrix.zig_target }} ${{ matrix.zig_target }} /tmp/zig-gnu - - - name: Build ${{ matrix.zig_target }} - env: - OPENSHELL_IMAGE_TAG: ${{ github.sha }} - run: | - set -euo pipefail - mise x -- rustup target add ${{ matrix.target }} - mise x -- cargo zigbuild --release --target ${{ matrix.zig_target }} -p openshell-server --bin openshell-gateway --features bundled-z3 - mkdir -p artifacts/bin - install -m 0755 target/${{ matrix.target }}/release/openshell-gateway artifacts/bin/openshell-gateway - - - name: Verify packaged binary - run: | - set -euo pipefail - OUTPUT="$(artifacts/bin/openshell-gateway --version)" - echo "$OUTPUT" - grep -q '^openshell-gateway ' <<<"$OUTPUT" - ldd artifacts/bin/openshell-gateway || true - if ldd artifacts/bin/openshell-gateway | grep -q 'libz3'; then - echo "gateway binary must not depend on shared libz3; build with bundled-z3" >&2 - exit 1 - fi - - - name: Verify glibc symbol floor - run: tasks/scripts/verify-glibc-symbols.sh 2.28 artifacts/bin/openshell-gateway - - - name: sccache stats - if: always() - run: mise x -- sccache --show-stats - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-gateway-${{ matrix.target }}.tar.gz \ - -C artifacts/bin openshell-gateway - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: gateway-binary-linux-${{ matrix.arch }} - path: artifacts/*.tar.gz - retention-days: 5 - - # --------------------------------------------------------------------------- - # Build standalone gateway binary (macOS aarch64 via osxcross) - # --------------------------------------------------------------------------- - build-gateway-binary-macos: - name: Build Gateway Binary (macOS) - needs: [compute-versions] - runs-on: linux-amd64-cpu8 - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - - name: Build macOS binary via Docker - run: | - set -euo pipefail - docker buildx build \ - --file deploy/docker/Dockerfile.gateway-macos \ - --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ - --build-arg OPENSHELL_IMAGE_TAG=dev \ - --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ - --target binary \ - --output type=local,dest=out/ \ - . - - - name: Verify packaged binary shape - run: | - set -euo pipefail - test -x out/openshell-gateway - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-gateway-aarch64-apple-darwin.tar.gz \ - -C out openshell-gateway - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: gateway-binary-macos - path: artifacts/*.tar.gz - retention-days: 5 - - # --------------------------------------------------------------------------- - # Build standalone supervisor binaries (Linux GNU — native on each arch) - # --------------------------------------------------------------------------- - build-supervisor-binary-linux: - name: Build Supervisor Binary (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - target: x86_64-unknown-linux-gnu - - arch: arm64 - runner: linux-arm64-cpu8 - target: aarch64-unknown-linux-gnu - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: supervisor-binary-gnu-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Patch workspace version - if: needs.compute-versions.outputs.cargo_version != '' - run: | - set -euo pipefail - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml - - - name: Build ${{ matrix.target }} - run: | - set -euo pipefail - mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-sandbox --bin openshell-sandbox - - - name: Verify packaged binary - run: | - set -euo pipefail - OUTPUT="$(target/${{ matrix.target }}/release/openshell-sandbox --version)" - echo "$OUTPUT" - grep -q '^openshell-sandbox ' <<<"$OUTPUT" - - - name: sccache stats - if: always() - run: mise x -- sccache --show-stats - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-sandbox-${{ matrix.target }}.tar.gz \ - -C target/${{ matrix.target }}/release openshell-sandbox - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: supervisor-binary-linux-${{ matrix.arch }} - path: artifacts/*.tar.gz - retention-days: 5 - - build-driver-vm-linux: - name: Build Driver VM Linux - needs: [compute-versions] - uses: ./.github/workflows/driver-vm-linux.yml - with: - cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} - image-tag: dev - checkout-ref: ${{ github.sha }} - secrets: inherit - - build-driver-vm-macos: - name: Build Driver VM macOS - needs: [compute-versions] - uses: ./.github/workflows/driver-vm-macos.yml - with: - cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} - image-tag: dev - checkout-ref: ${{ github.sha }} - secrets: inherit build-deb: name: Build Debian Packages - needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-driver-vm-linux] + needs: [compute-versions, build-cli, build-gateway, build-vm-driver] uses: ./.github/workflows/deb-package.yml with: deb-version: ${{ needs.compute-versions.outputs.deb_version }} @@ -596,7 +223,7 @@ jobs: build-snap: name: Build Snap - needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-supervisor-binary-linux] + needs: [compute-versions, build-cli, build-gateway, build-sandbox] uses: ./.github/workflows/snap-package.yml with: checkout-ref: ${{ github.sha }} @@ -607,7 +234,7 @@ jobs: build-rpm: name: Build RPM Packages - needs: [compute-versions, build-cli-linux, build-gateway-binary-linux] + needs: [compute-versions, build-cli, build-gateway] uses: ./.github/workflows/rpm-package.yml with: checkout-ref: ${{ github.sha }} @@ -618,7 +245,7 @@ jobs: smoke-linux-dev-artifacts: name: Smoke Linux Dev Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheel] + needs: [package-binaries, build-deb, build-rpm, build-python-wheel] timeout-minutes: 20 strategy: fail-fast: false @@ -712,7 +339,7 @@ jobs: # --------------------------------------------------------------------------- release-dev: name: Release Dev - needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheel, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-dev-artifacts] + needs: [compute-versions, package-binaries, build-python-wheel, docker-e2e, podman-e2e, vm-e2e, build-deb, build-rpm, build-snap, smoke-linux-dev-artifacts] runs-on: linux-amd64-cpu8 timeout-minutes: 10 permissions: @@ -839,8 +466,8 @@ jobs: openshell-gateway-aarch64-apple-darwin.tar.gz > openshell-gateway-checksums-sha256.txt cat openshell-gateway-checksums-sha256.txt sha256sum \ - openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz \ - openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-checksums-sha256.txt + openshell-sandbox-x86_64-unknown-linux-musl.tar.gz \ + openshell-sandbox-aarch64-unknown-linux-musl.tar.gz > openshell-sandbox-checksums-sha256.txt cat openshell-sandbox-checksums-sha256.txt - name: Generate Homebrew formula @@ -945,8 +572,8 @@ jobs: release/openshell-gateway-x86_64-unknown-linux-gnu.tar.gz release/openshell-gateway-aarch64-unknown-linux-gnu.tar.gz release/openshell-gateway-aarch64-apple-darwin.tar.gz - release/openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz - release/openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-x86_64-unknown-linux-musl.tar.gz + release/openshell-sandbox-aarch64-unknown-linux-musl.tar.gz release/openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-apple-darwin.tar.gz diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 74be300b0c..92eb4adf14 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -73,46 +73,126 @@ jobs: echo "semver=${RELEASE_TAG#v}" >> "$GITHUB_OUTPUT" echo "source_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + build-cli: + needs: compute-versions + permissions: + contents: read + uses: ./.github/workflows/build-cli-binaries.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: ${{ needs.compute-versions.outputs.semver }} + checkout-ref: ${{ inputs.tag || github.ref }} + secrets: inherit + build-gateway: - needs: [compute-versions] + needs: compute-versions + permissions: + contents: read + uses: ./.github/workflows/build-gateway-binaries.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: ${{ needs.compute-versions.outputs.semver }} + checkout-ref: ${{ inputs.tag || github.ref }} + secrets: inherit + + build-sandbox: + needs: compute-versions + permissions: + contents: read + uses: ./.github/workflows/build-sandbox-binaries.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: ${{ needs.compute-versions.outputs.semver }} + checkout-ref: ${{ inputs.tag || github.ref }} + secrets: inherit + + build-vm-driver: + needs: [compute-versions, build-sandbox] + permissions: + contents: read + uses: ./.github/workflows/build-vm-driver.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: ${{ needs.compute-versions.outputs.semver }} + checkout-ref: ${{ inputs.tag || github.ref }} + secrets: inherit + + package-binaries: + needs: [build-cli, build-gateway, build-sandbox, build-vm-driver] + permissions: + actions: read + contents: read + uses: ./.github/workflows/package-release-binaries.yml + + build-gateway-image: + needs: [compute-versions, build-gateway] + permissions: + contents: read + packages: write uses: ./.github/workflows/docker-build.yml with: component: gateway - cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + binary: openshell-gateway + target-suffix: unknown-linux-gnu image-tag: ${{ needs.compute-versions.outputs.source_sha }} checkout-ref: ${{ inputs.tag || github.ref }} - auditable: true + secrets: inherit - build-supervisor: - needs: [compute-versions] + build-supervisor-image: + needs: [compute-versions, build-sandbox] + permissions: + contents: read + packages: write uses: ./.github/workflows/docker-build.yml with: component: supervisor - cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + binary: openshell-sandbox + target-suffix: unknown-linux-musl image-tag: ${{ needs.compute-versions.outputs.source_sha }} checkout-ref: ${{ inputs.tag || github.ref }} - auditable: true + secrets: inherit - e2e: - needs: [compute-versions, build-gateway, build-supervisor] + docker-e2e: + needs: [compute-versions, build-cli, build-gateway, build-supervisor-image] permissions: actions: read contents: read packages: read - uses: ./.github/workflows/e2e-test.yml + uses: ./.github/workflows/e2e-docker-test.yml with: image-tag: ${{ needs.compute-versions.outputs.source_sha }} checkout-ref: ${{ inputs.tag || github.ref }} runner: linux-arm64-cpu8 + podman-e2e: + needs: [compute-versions, build-cli, build-gateway, build-supervisor-image] + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-podman-test.yml + with: + image-tag: ${{ needs.compute-versions.outputs.source_sha }} + checkout-ref: ${{ inputs.tag || github.ref }} + + vm-e2e: + needs: [compute-versions, build-cli, build-gateway, build-vm-driver] + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-vm-test.yml + with: + checkout-ref: ${{ inputs.tag || github.ref }} + tag-ghcr-release: name: Tag GHCR Images for Release - needs: [compute-versions, build-gateway, build-supervisor, release] + needs: [compute-versions, build-gateway-image, build-supervisor-image, release] runs-on: linux-amd64-cpu8 timeout-minutes: 10 steps: - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin - name: Tag images with version and latest run: | @@ -170,461 +250,9 @@ jobs: path: target/wheels/*.whl retention-days: 5 - # --------------------------------------------------------------------------- - # Build CLI binaries (Linux musl — static, native on each arch) - # - # Builds run directly on the CI host (glibc Ubuntu). Zig provides musl - # C/C++ toolchains for bundled-z3 and ring, and is also used as the linker. - # --------------------------------------------------------------------------- - build-cli-linux: - name: Build CLI (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - target: x86_64-unknown-linux-musl - zig_target: x86_64-linux-musl - - arch: arm64 - runner: linux-arm64-cpu8 - target: aarch64-unknown-linux-musl - zig_target: aarch64-linux-musl - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.tag || github.ref }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: cli-musl-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Add Rust musl target - run: mise x -- rustup target add ${{ matrix.target }} - - - name: Set up zig musl wrappers - run: | - set -euo pipefail - ZIG="$(mise which zig)" - ZIG_TARGET="${{ matrix.zig_target }}" - mkdir -p /tmp/zig-musl - - # cc-rs injects --target= (for example - # aarch64-unknown-linux-musl), which zig does not parse. Strip any - # caller-provided --target and use the wrapper's zig-native target. - for tool in cc c++; do - printf '#!/bin/bash\nargs=()\nfor arg in "$@"; do\n case "$arg" in\n --target=*) ;;\n *) args+=("$arg") ;;\n esac\ndone\nexec "%s" %s --target=%s "${args[@]}"\n' \ - "$ZIG" "$tool" "$ZIG_TARGET" > "/tmp/zig-musl/${tool}" - chmod +x "/tmp/zig-musl/${tool}" - done - - TARGET_ENV=$(echo "${{ matrix.target }}" | tr '-' '_') - TARGET_ENV_UPPER=${TARGET_ENV^^} - - # Use zig for C/C++ compilation and final linking. - echo "CC_${TARGET_ENV}=/tmp/zig-musl/cc" >> "$GITHUB_ENV" - echo "CXX_${TARGET_ENV}=/tmp/zig-musl/c++" >> "$GITHUB_ENV" - echo "CARGO_TARGET_${TARGET_ENV_UPPER}_LINKER=/tmp/zig-musl/cc" >> "$GITHUB_ENV" - - # Let zig own CRT/startfiles to avoid duplicate _start symbols. - echo "CARGO_TARGET_${TARGET_ENV_UPPER}_RUSTFLAGS=-Clink-self-contained=no" >> "$GITHUB_ENV" - - # z3 built with zig c++ uses libc++ symbols (std::__1::*). - # Override z3-sys default (stdc++) so Rust links the matching runtime. - echo "CXXSTDLIB=c++" >> "$GITHUB_ENV" - - - name: Patch workspace version - if: needs.compute-versions.outputs.cargo_version != '' - run: | - set -euo pipefail - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml - - - name: Build ${{ matrix.target }} - run: mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-cli - - - name: sccache stats - if: always() - run: mise x -- sccache --show-stats - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-${{ matrix.target }}.tar.gz \ - -C target/${{ matrix.target }}/release openshell - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: cli-linux-${{ matrix.arch }} - path: artifacts/*.tar.gz - retention-days: 5 - - # --------------------------------------------------------------------------- - # Build CLI binary (macOS aarch64 via osxcross) - # --------------------------------------------------------------------------- - build-cli-macos: - name: Build CLI (macOS) - needs: [compute-versions] - runs-on: linux-amd64-cpu8 - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.tag || github.ref }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - - name: Build macOS binary via Docker - run: | - set -euo pipefail - docker buildx build \ - --file deploy/docker/Dockerfile.cli-macos \ - --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ - --build-arg OPENSHELL_IMAGE_TAG="${{ needs.compute-versions.outputs.semver }}" \ - --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ - --target binary \ - --output type=local,dest=out/ \ - . - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-aarch64-apple-darwin.tar.gz \ - -C out openshell - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: cli-macos - path: artifacts/*.tar.gz - retention-days: 5 - - # --------------------------------------------------------------------------- - # Build standalone gateway binaries (Linux GNU — glibc 2.28 floor) - # --------------------------------------------------------------------------- - build-gateway-binary-linux: - name: Build Gateway Binary (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - target: x86_64-unknown-linux-gnu - zig_target: x86_64-unknown-linux-gnu.2.28 - - arch: arm64 - runner: linux-arm64-cpu8 - target: aarch64-unknown-linux-gnu - zig_target: aarch64-unknown-linux-gnu.2.28 - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.tag || github.ref }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: gateway-binary-gnu-${{ matrix.arch }}-zig-wrapper-${{ hashFiles('tasks/scripts/setup-zig-cc-wrapper.sh') }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Patch workspace version - if: needs.compute-versions.outputs.cargo_version != '' - run: | - set -euo pipefail - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml - - - name: Set up Zig C/C++ wrappers - run: tasks/scripts/setup-zig-cc-wrapper.sh ${{ matrix.zig_target }} ${{ matrix.zig_target }} /tmp/zig-gnu - - - name: Build ${{ matrix.zig_target }} - env: - OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.source_sha }} - run: | - set -euo pipefail - mise x -- rustup target add ${{ matrix.target }} - mise x -- cargo zigbuild --release --target ${{ matrix.zig_target }} -p openshell-server --bin openshell-gateway --features bundled-z3 - mkdir -p artifacts/bin - install -m 0755 target/${{ matrix.target }}/release/openshell-gateway artifacts/bin/openshell-gateway - - - name: Verify packaged binary - run: | - set -euo pipefail - OUTPUT="$(artifacts/bin/openshell-gateway --version)" - echo "$OUTPUT" - grep -q '^openshell-gateway ' <<<"$OUTPUT" - ldd artifacts/bin/openshell-gateway || true - if ldd artifacts/bin/openshell-gateway | grep -q 'libz3'; then - echo "gateway binary must not require shared libz3; keep z3 bundled for portable release artifacts" >&2 - exit 1 - fi - - - name: Verify glibc symbol floor - run: tasks/scripts/verify-glibc-symbols.sh 2.28 artifacts/bin/openshell-gateway - - - name: sccache stats - if: always() - run: mise x -- sccache --show-stats - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-gateway-${{ matrix.target }}.tar.gz \ - -C artifacts/bin openshell-gateway - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: gateway-binary-linux-${{ matrix.arch }} - path: artifacts/*.tar.gz - retention-days: 5 - - # --------------------------------------------------------------------------- - # Build standalone supervisor binaries (Linux GNU — native on each arch) - # --------------------------------------------------------------------------- - build-supervisor-binary-linux: - name: Build Supervisor Binary (Linux ${{ matrix.arch }}) - needs: [compute-versions] - strategy: - matrix: - include: - - arch: amd64 - runner: linux-amd64-cpu8 - target: x86_64-unknown-linux-gnu - - arch: arm64 - runner: linux-arm64-cpu8 - target: aarch64-unknown-linux-gnu - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.tag || github.ref }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Install tools - run: mise install --locked - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: supervisor-binary-gnu-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Patch workspace version - if: needs.compute-versions.outputs.cargo_version != '' - run: | - set -euo pipefail - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml - - - name: Build ${{ matrix.target }} - run: | - set -euo pipefail - mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-sandbox --bin openshell-sandbox - - - name: Verify packaged binary - run: | - set -euo pipefail - OUTPUT="$(target/${{ matrix.target }}/release/openshell-sandbox --version)" - echo "$OUTPUT" - grep -q '^openshell-sandbox ' <<<"$OUTPUT" - - - name: sccache stats - if: always() - run: mise x -- sccache --show-stats - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-sandbox-${{ matrix.target }}.tar.gz \ - -C target/${{ matrix.target }}/release openshell-sandbox - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: supervisor-binary-linux-${{ matrix.arch }} - path: artifacts/*.tar.gz - retention-days: 5 - - # --------------------------------------------------------------------------- - # Build standalone gateway binary (macOS aarch64 via osxcross) - # --------------------------------------------------------------------------- - build-gateway-binary-macos: - name: Build Gateway Binary (macOS) - needs: [compute-versions] - runs-on: linux-amd64-cpu8 - timeout-minutes: 60 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.tag || github.ref }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx - - - name: Build macOS binary via Docker - run: | - set -euo pipefail - docker buildx build \ - --file deploy/docker/Dockerfile.gateway-macos \ - --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ - --build-arg OPENSHELL_IMAGE_TAG="${{ needs.compute-versions.outputs.semver }}" \ - --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ - --target binary \ - --output type=local,dest=out/ \ - . - - - name: Verify packaged binary shape - run: | - set -euo pipefail - test -x out/openshell-gateway - - - name: Package binary - run: | - set -euo pipefail - mkdir -p artifacts - tar -czf artifacts/openshell-gateway-aarch64-apple-darwin.tar.gz \ - -C out openshell-gateway - ls -lh artifacts/ - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: gateway-binary-macos - path: artifacts/*.tar.gz - retention-days: 5 - - build-driver-vm-linux: - name: Build Driver VM Linux - needs: [compute-versions] - uses: ./.github/workflows/driver-vm-linux.yml - with: - cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} - image-tag: ${{ needs.compute-versions.outputs.semver }} - checkout-ref: ${{ inputs.tag || github.ref }} - secrets: inherit - - build-driver-vm-macos: - name: Build Driver VM macOS - needs: [compute-versions] - uses: ./.github/workflows/driver-vm-macos.yml - with: - cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} - image-tag: ${{ needs.compute-versions.outputs.semver }} - checkout-ref: ${{ inputs.tag || github.ref }} - secrets: inherit - build-deb: name: Build Debian Packages - needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-driver-vm-linux] + needs: [compute-versions, build-cli, build-gateway, build-vm-driver] uses: ./.github/workflows/deb-package.yml with: deb-version: ${{ needs.compute-versions.outputs.deb_version }} @@ -633,7 +261,7 @@ jobs: build-snap: name: Build Snap - needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-supervisor-binary-linux] + needs: [compute-versions, build-cli, build-gateway, build-sandbox] uses: ./.github/workflows/snap-package.yml with: checkout-ref: ${{ inputs.tag || github.ref }} @@ -644,7 +272,7 @@ jobs: build-rpm: name: Build RPM Packages - needs: [compute-versions, build-cli-linux, build-gateway-binary-linux] + needs: [compute-versions, build-cli, build-gateway] uses: ./.github/workflows/rpm-package.yml with: checkout-ref: ${{ inputs.tag || github.ref }} @@ -655,7 +283,7 @@ jobs: smoke-linux-release-artifacts: name: Smoke Linux Release Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheel] + needs: [package-binaries, build-deb, build-rpm, build-python-wheel] timeout-minutes: 20 strategy: fail-fast: false @@ -792,7 +420,7 @@ jobs: # --------------------------------------------------------------------------- release: name: Release - needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheel, e2e, build-driver-vm-linux, build-driver-vm-macos, build-deb, build-rpm, build-snap, smoke-linux-release-artifacts] + needs: [compute-versions, package-binaries, build-python-wheel, docker-e2e, podman-e2e, vm-e2e, build-deb, build-rpm, build-snap, smoke-linux-release-artifacts] runs-on: linux-amd64-cpu8 timeout-minutes: 10 permissions: @@ -896,8 +524,8 @@ jobs: openshell-gateway-aarch64-apple-darwin.tar.gz > openshell-gateway-checksums-sha256.txt cat openshell-gateway-checksums-sha256.txt sha256sum \ - openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz \ - openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-checksums-sha256.txt + openshell-sandbox-x86_64-unknown-linux-musl.tar.gz \ + openshell-sandbox-aarch64-unknown-linux-musl.tar.gz > openshell-sandbox-checksums-sha256.txt cat openshell-sandbox-checksums-sha256.txt - name: Generate Homebrew formula @@ -966,8 +594,8 @@ jobs: release/openshell-gateway-x86_64-unknown-linux-gnu.tar.gz release/openshell-gateway-aarch64-unknown-linux-gnu.tar.gz release/openshell-gateway-aarch64-apple-darwin.tar.gz - release/openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz - release/openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-x86_64-unknown-linux-musl.tar.gz + release/openshell-sandbox-aarch64-unknown-linux-musl.tar.gz release/openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-apple-darwin.tar.gz diff --git a/.github/workflows/rpm-package.yml b/.github/workflows/rpm-package.yml index 591d10e619..8701644585 100644 --- a/.github/workflows/rpm-package.yml +++ b/.github/workflows/rpm-package.yml @@ -68,21 +68,19 @@ jobs: - name: Download CLI artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: cli-linux-${{ matrix.artifact_arch }} - path: package-input/ + name: openshell-${{ matrix.cli_target }} + path: package-binaries/ - name: Download gateway artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: gateway-binary-linux-${{ matrix.artifact_arch }} - path: package-input/ + name: openshell-gateway-${{ matrix.gnu_target }} + path: package-binaries/ - - name: Extract package inputs + - name: Configure package inputs run: | set -euo pipefail - mkdir -p package-binaries - tar -xzf "package-input/openshell-${{ matrix.cli_target }}.tar.gz" -C package-binaries - tar -xzf "package-input/openshell-gateway-${{ matrix.gnu_target }}.tar.gz" -C package-binaries + chmod +x package-binaries/openshell{,-gateway} ls -lah package-binaries - name: Mark workspace safe for git diff --git a/.github/workflows/rust-native-build.yml b/.github/workflows/rust-native-build.yml deleted file mode 100644 index 24124d0ae4..0000000000 --- a/.github/workflows/rust-native-build.yml +++ /dev/null @@ -1,383 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Rust Image Binary Build (openshell-gateway / openshell-sandbox / openshell-cli) - -# Build Rust binaries per Linux architecture before the Docker image build -# consumes them as prebuilt artifacts. Gateway images use GNU-linked binaries -# for the NVIDIA distroless C/C++ runtime; supervisor and cli images use static -# binaries so the final image can remain scratch. Gateway GNU binaries are -# built with an explicit glibc 2.28 floor so image, package, and tarball -# artifacts share the same host portability contract. -# -# The supervisor libc is selectable via the `supervisor-libc` input (musl or -# glibc-static). Both variants are fully static and are verified as such, -# because the supervisor is executed from inside arbitrary sandbox images. - -on: - workflow_call: - inputs: - component: - description: "Binary component to build (gateway, sandbox, or cli)" - required: true - type: string - arch: - description: "Linux architecture to build (amd64 or arm64)" - required: true - type: string - supervisor-libc: - description: "libc variant for the sandbox component (musl or glibc-static)" - required: false - type: string - default: "musl" - cargo-version: - description: "Pre-computed cargo version (skips internal git-based computation)" - required: false - type: string - default: "" - features: - description: "Cargo features to enable" - required: false - type: string - default: "" - # Branch E2E and release gateway/supervisor image builds opt in. - # CLI images and standalone glibc-static supervisor validation keep false. - auditable: - description: "Embed cargo-auditable dependency metadata in the binary" - required: false - type: boolean - default: false - retention-days: - description: "Artifact retention period" - required: false - type: number - default: 5 - artifact-name: - description: "Artifact name override" - required: false - type: string - default: "" - checkout-ref: - description: "Git ref to check out for build inputs (defaults to the workflow SHA)" - required: false - type: string - default: "" - image-tag: - description: "Supervisor image tag to bake into gateway binaries" - required: false - type: string - default: "" -permissions: - contents: read - packages: read - -env: - CARGO_TERM_COLOR: always - CARGO_INCREMENTAL: "0" - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Route sccache (already RUSTC_WRAPPER in mise.toml) to the GHA cache - # backend instead of the EKS memcached used by ARC. - SCCACHE_GHA_ENABLED: "true" - -defaults: - run: - shell: bash - -jobs: - rust-native-build: - name: ${{ inputs.component }} (${{ inputs.arch }}) - runs-on: ${{ inputs.arch == 'arm64' && 'linux-arm64-cpu8' || 'linux-amd64-cpu8' }} - timeout-minutes: 60 - env: - COMPONENT: ${{ inputs.component }} - ARCH: ${{ inputs.arch }} - FEATURES: ${{ inputs.features }} - SUPERVISOR_LIBC: ${{ inputs['supervisor-libc'] }} - # Partition the GHA sccache cache per (component, arch). Without this, - # concurrent jobs collide on the same cache key and later-starting - # writers hit 409 Conflict. The sandbox component also partitions per - # libc variant so musl and glibc-static builds do not evict each other. - SCCACHE_GHA_VERSION: ${{ inputs.component }}-${{ inputs.arch }}${{ inputs.component == 'sandbox' && format('-{0}', inputs['supervisor-libc']) || '' }} - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs['checkout-ref'] || github.sha }} - fetch-depth: 0 - - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Fetch tags - run: git fetch --tags --force - - - name: Configure GHA sccache backend - # Exposes ACTIONS_CACHE_URL / ACTIONS_RUNTIME_TOKEN before `mise install` - # compiles cargo-installed tools through RUSTC_WRAPPER=sccache. - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - - name: Install tools - run: mise install --locked - - - name: Resolve build target - id: target - run: | - set -euo pipefail - - case "$COMPONENT" in - gateway) - crate=openshell-server - binary=openshell-gateway - zig_target= - ;; - sandbox) - crate=openshell-sandbox - binary=openshell-sandbox - zig_target= - ;; - cli) - crate=openshell-cli - binary=openshell - zig_target= - ;; - *) - echo "unsupported component: $COMPONENT" >&2 - exit 1 - ;; - esac - - # The sandbox binary must stay fully static. musl gets there via the - # musl target; glibc-static uses the GNU target with +crt-static and - # relies on this job running natively on the target architecture, - # because zig cannot statically link glibc. - static_libc=musl - if [[ "$COMPONENT" == "sandbox" ]]; then - case "$SUPERVISOR_LIBC" in - musl) static_libc=musl ;; - glibc-static) static_libc=gnu ;; - *) - echo "unsupported supervisor-libc: $SUPERVISOR_LIBC (expected musl or glibc-static)" >&2 - exit 1 - ;; - esac - fi - - case "$ARCH" in - amd64) - if [[ "$COMPONENT" == "sandbox" && "$static_libc" == "gnu" ]]; then - target=x86_64-unknown-linux-gnu - zig_target= - elif [[ "$COMPONENT" == "sandbox" || "$COMPONENT" == "cli" ]]; then - target=x86_64-unknown-linux-musl - zig_target=x86_64-linux-musl - else - target=x86_64-unknown-linux-gnu - zig_target=x86_64-unknown-linux-gnu.2.28 - fi - ;; - arm64) - if [[ "$COMPONENT" == "sandbox" && "$static_libc" == "gnu" ]]; then - target=aarch64-unknown-linux-gnu - zig_target= - elif [[ "$COMPONENT" == "sandbox" || "$COMPONENT" == "cli" ]]; then - target=aarch64-unknown-linux-musl - zig_target=aarch64-linux-musl - else - target=aarch64-unknown-linux-gnu - zig_target=aarch64-unknown-linux-gnu.2.28 - fi - ;; - *) - echo "unsupported arch: $ARCH" >&2 - exit 1 - ;; - esac - - { - echo "crate=$crate" - echo "binary=$binary" - echo "target=$target" - echo "zig_target=$zig_target" - } >> "$GITHUB_OUTPUT" - - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 - with: - shared-key: rust-native-${{ inputs.component }}-${{ inputs.arch }}${{ inputs.component == 'sandbox' && format('-{0}', inputs['supervisor-libc']) || '' }}-zig-wrapper-${{ hashFiles('tasks/scripts/setup-zig-cc-wrapper.sh') }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Compute cargo version - id: version - run: | - set -euo pipefail - if [[ -n "${{ inputs['cargo-version'] }}" ]]; then - echo "cargo_version=${{ inputs['cargo-version'] }}" >> "$GITHUB_OUTPUT" - else - echo "cargo_version=$(uv run python tasks/scripts/release.py get-version --cargo)" >> "$GITHUB_OUTPUT" - fi - - - name: Patch workspace version - if: steps.version.outputs.cargo_version != '' - run: | - set -euo pipefail - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ steps.version.outputs.cargo_version }}"'"/}' Cargo.toml - - - name: Set up zig musl wrappers - if: contains(steps.target.outputs.target, 'musl') - run: | - set -euo pipefail - ZIG="$(mise which zig)" - ZIG_TARGET="${{ steps.target.outputs.zig_target }}" - mkdir -p /tmp/zig-musl - - # cc-rs injects --target=, which zig does not parse. - # Strip caller-provided --target and use the wrapper's zig target. - for tool in cc c++; do - printf '#!/bin/bash\nargs=()\nfor arg in "$@"; do\n case "$arg" in\n --target=*) ;;\n *) args+=("$arg") ;;\n esac\ndone\nexec "%s" %s --target=%s "${args[@]}"\n' \ - "$ZIG" "$tool" "$ZIG_TARGET" > "/tmp/zig-musl/${tool}" - chmod +x "/tmp/zig-musl/${tool}" - done - - TARGET_ENV=$(echo "${{ steps.target.outputs.target }}" | tr '-' '_') - TARGET_ENV_UPPER=${TARGET_ENV^^} - - echo "CC_${TARGET_ENV}=/tmp/zig-musl/cc" >> "$GITHUB_ENV" - echo "CXX_${TARGET_ENV}=/tmp/zig-musl/c++" >> "$GITHUB_ENV" - echo "CARGO_TARGET_${TARGET_ENV_UPPER}_LINKER=/tmp/zig-musl/cc" >> "$GITHUB_ENV" - echo "CARGO_TARGET_${TARGET_ENV_UPPER}_RUSTFLAGS=-Clink-self-contained=no" >> "$GITHUB_ENV" - - - name: Set up zig glibc wrappers - if: inputs.component == 'gateway' - run: tasks/scripts/setup-zig-cc-wrapper.sh "${{ steps.target.outputs.zig_target }}" "${{ steps.target.outputs.zig_target }}" /tmp/zig-gnu - - - name: Build ${{ steps.target.outputs.binary }} (${{ steps.target.outputs.zig_target || steps.target.outputs.target }}) - env: - # Preserve the release-codegen setting used by the old Dockerfile - # Rust build path so image artifacts keep the same release profile. - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1" - OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} - run: | - set -euo pipefail - # z3 built with zig c++ uses libc++ symbols (std::__1::*). - # Override z3-sys default (stdc++) so Rust links the matching runtime. - if [[ "${{ inputs.component }}" == "cli" ]]; then - echo "CXXSTDLIB=c++" >> "$GITHUB_ENV" - fi - - mise x -- rustup target add "${{ steps.target.outputs.target }}" - - cargo_cmd=(cargo build) - cargo_env=() - build_target="${{ steps.target.outputs.target }}" - args=() - - if [[ "${{ inputs.component }}" == "gateway" ]]; then - cargo_cmd=(cargo zigbuild) - build_target="${{ steps.target.outputs.zig_target }}" - args+=(--features bundled-z3) - elif [[ "${{ inputs.component }}" == "sandbox" && "$SUPERVISOR_LIBC" == "glibc-static" ]]; then - # Static glibc requires the native toolchain's libc.a (build-essential - # in the CI image); cargo-zigbuild is not usable here because zig - # accepts -static for *-linux-gnu and links dynamically anyway. - export RUSTFLAGS="${RUSTFLAGS:-} -C target-feature=+crt-static" - fi - if [[ "${{ inputs.auditable }}" == "true" ]]; then - cargo_cmd=("${cargo_cmd[0]}" auditable "${cargo_cmd[@]:1}") - # mise.toml injects RUSTC_WRAPPER=sccache. Unset it after mise has - # constructed the environment so it cannot wrap cargo-auditable's - # RUSTC_WORKSPACE_WRAPPER and misidentify that wrapper as rustc. - cargo_env=(env -u RUSTC_WRAPPER) - fi - args+=( - --release - --target "$build_target" - -p "${{ steps.target.outputs.crate }}" - --bin "${{ steps.target.outputs.binary }}" - ) - if [[ -n "$FEATURES" ]]; then - args+=(--features "$FEATURES") - fi - if [[ -n "${{ steps.version.outputs.cargo_version }}" ]]; then - export GIT_DIR=/nonexistent - fi - mise x -- "${cargo_env[@]}" "${cargo_cmd[@]}" "${args[@]}" - - - name: Verify packaged binary - run: | - set -euo pipefail - BIN="target/${{ steps.target.outputs.target }}/release/${{ steps.target.outputs.binary }}" - OUTPUT="$("$BIN" --version)" - echo "$OUTPUT" - grep -q "^${{ steps.target.outputs.binary }} " <<<"$OUTPUT" - # Record linkage so image runtime drift is visible in logs. - ldd --version - ldd "$BIN" || true - if [[ "${{ inputs.component }}" == "gateway" ]] && ldd "$BIN" | grep -q 'libz3'; then - echo "gateway binary must not depend on shared libz3; enable bundled-z3 for image artifacts" >&2 - exit 1 - fi - - - name: Verify auditable dependency metadata - if: inputs.auditable - run: | - set -euo pipefail - BIN="target/${{ steps.target.outputs.target }}/release/${{ steps.target.outputs.binary }}" - cargo_packages="$( - SYFT_CHECK_FOR_APP_UPDATE=false \ - mise x -- syft "file:$BIN" -o cyclonedx-json | - jq '[.components[]? | select((.purl // "") | startswith("pkg:cargo/"))] | length' - )" - if [[ "$cargo_packages" -eq 0 ]]; then - echo "Syft did not decode any Cargo packages from $BIN" >&2 - exit 1 - fi - echo "auditable dependency metadata: $cargo_packages Cargo packages" - - - name: Verify glibc symbol floor - if: inputs.component == 'gateway' - run: | - set -euo pipefail - BIN="target/${{ steps.target.outputs.target }}/release/${{ steps.target.outputs.binary }}" - tasks/scripts/verify-glibc-symbols.sh 2.28 "$BIN" - - - name: Verify static linkage - if: inputs.component == 'sandbox' - run: | - set -euo pipefail - BIN="target/${{ steps.target.outputs.target }}/release/${{ steps.target.outputs.binary }}" - tasks/scripts/verify-static-binary.sh "$BIN" - - - name: Stage binary for prebuilt layout - run: | - set -euo pipefail - STAGE="prebuilt-binaries/$ARCH" - mkdir -p "$STAGE" - install -m 0755 \ - "target/${{ steps.target.outputs.target }}/release/${{ steps.target.outputs.binary }}" \ - "$STAGE/${{ steps.target.outputs.binary }}" - ls -lh "$STAGE/" - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ${{ inputs['artifact-name'] != '' && inputs['artifact-name'] || format('rust-binary-{0}-linux-{1}', inputs.component, inputs.arch) }} - path: prebuilt-binaries/${{ inputs.arch }}/${{ steps.target.outputs.binary }} - retention-days: ${{ inputs['retention-days'] }} - if-no-files-found: error - - - name: sccache stats - if: always() - run: | - set +e - stats_bin="${SCCACHE_PATH:-sccache}" - "$stats_bin" --show-stats - status=$? - if [[ $status -ne 0 ]]; then - echo "::warning::sccache stats unavailable (exit $status)" - fi - exit 0 diff --git a/.github/workflows/snap-package.yml b/.github/workflows/snap-package.yml index ed248b79e5..6de2a587f2 100644 --- a/.github/workflows/snap-package.yml +++ b/.github/workflows/snap-package.yml @@ -37,8 +37,10 @@ jobs: matrix: include: - arch: amd64 + rust_arch: x86_64 runner: linux-amd64-cpu8 - arch: arm64 + rust_arch: aarch64 runner: linux-arm64-cpu8 runs-on: ${{ matrix.runner }} timeout-minutes: 60 @@ -79,35 +81,27 @@ jobs: - name: Download prebuilt CLI binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: cli-linux-${{ matrix.arch }} + name: openshell-${{ matrix.rust_arch }}-unknown-linux-musl path: prebuilt/cli - name: Download prebuilt gateway binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: gateway-binary-linux-${{ matrix.arch }} + name: openshell-gateway-${{ matrix.rust_arch }}-unknown-linux-gnu path: prebuilt/gateway - name: Download prebuilt sandbox binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: supervisor-binary-linux-${{ matrix.arch }} + name: openshell-sandbox-${{ matrix.rust_arch }}-unknown-linux-musl path: prebuilt/sandbox - - name: Extract prebuilt binaries + - name: Configure prebuilt binaries run: | set -euo pipefail - mkdir -p prebuilt/{cli,gateway,sandbox} - - for d in cli gateway sandbox; do - for tarball in prebuilt/$d/*.tar.gz; do - if [ -f "$tarball" ]; then - tar -xzf "$tarball" -C "prebuilt/$d" - else - echo "WARNING: no tarball found in prebuilt/$d/" >&2 - fi - done - done + chmod +x prebuilt/cli/openshell + chmod +x prebuilt/gateway/openshell-gateway + chmod +x prebuilt/sandbox/openshell-sandbox ls -laR prebuilt/ - name: Prepare snap build directory @@ -169,6 +163,7 @@ jobs: - name: Upload snap to Snap Store env: SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.publish-credentials }} + INPUTS_UPLOAD_CHANNEL: ${{ inputs.upload-channel }} run: | set -euo pipefail SNAP_FILE="${{ steps.capture.outputs.snap-file }}" @@ -182,5 +177,5 @@ jobs: COMPONENT_ARGS+=(--component "$comp") done - echo "Uploading $SNAP_FILE to ${{ inputs.upload-channel }}" - snapcraft upload --release "${{ inputs.upload-channel }}" "$SNAP_FILE" "${COMPONENT_ARGS[@]}" + echo "Uploading $SNAP_FILE to ${INPUTS_UPLOAD_CHANNEL}" + snapcraft upload --release "${INPUTS_UPLOAD_CHANNEL}" "$SNAP_FILE" "${COMPONENT_ARGS[@]}" diff --git a/.github/workflows/supervisor-static-validate.yml b/.github/workflows/supervisor-static-validate.yml deleted file mode 100644 index 3b460a4588..0000000000 --- a/.github/workflows/supervisor-static-validate.yml +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Supervisor Static Linkage Validation - -# The glibc-static supervisor variant (SUPERVISOR_LIBC=glibc-static) has no -# other CI caller: docker-build.yml builds the default musl variant, so the -# GNU + crt-static build branch and its native-only, per-arch requirements are -# never exercised by image or release CI. Build the variant here on both -# architectures so it cannot regress unnoticed. rust-native-build.yml runs -# verify-static-binary.sh for the sandbox component, which fails the job on any -# dynamic linkage. -# -# Linkage can change from a new/updated dependency or a source change, not just -# from the build scripts, so the push path filters cover the workspace manifests -# and crate sources in addition to the build tooling. A nightly schedule is the -# unfiltered backstop for anything the filters miss. -# -# rust-native-build.yml runs on NVIDIA self-hosted runners, which reject jobs -# triggered by `pull_request`. This workflow therefore follows the repo's -# self-hosted convention (see branch-checks.yml / branch-e2e.yml): validate in -# the merge queue (pre-merge), on push to main (post-merge), nightly, and on -# demand — never on `pull_request`. - -on: - merge_group: - types: [checks_requested] - push: - branches: [main] - paths: - - "Cargo.toml" - - "Cargo.lock" - - "crates/**" - - "rust-toolchain.toml" - - "mise.toml" - - "mise.lock" - - ".cargo/config.toml" - - "tasks/scripts/stage-prebuilt-binaries.sh" - - "tasks/scripts/verify-static-binary.sh" - - ".github/workflows/rust-native-build.yml" - - ".github/workflows/supervisor-static-validate.yml" - schedule: - # Nightly (04:17 UTC) unfiltered run so a linkage regression cannot slip - # through the path filters unnoticed. Schedules run only on the default branch. - - cron: "17 4 * * *" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -permissions: - contents: read - packages: read - -jobs: - glibc-static: - name: glibc-static supervisor (${{ matrix.arch }}) - strategy: - fail-fast: false - matrix: - arch: [amd64, arm64] - uses: ./.github/workflows/rust-native-build.yml - with: - component: sandbox - arch: ${{ matrix.arch }} - supervisor-libc: glibc-static - artifact-name: supervisor-glibc-static-${{ matrix.arch }} - retention-days: 1 - secrets: inherit diff --git a/CI.md b/CI.md index c3d33622f7..2120fa546b 100644 --- a/CI.md +++ b/CI.md @@ -14,14 +14,14 @@ Merge queue validation is a second integration gate for `main`. After a PR has p Three opt-in labels enable the long-running E2E suites: -- `test:e2e` runs the standard Docker, Kubernetes, and VM E2E - suites in `Branch E2E Checks` +- `test:e2e` runs the Docker, rootless Podman, Kubernetes, and VM E2E suites + with both managed and standalone compute drivers in `Branch E2E Checks` - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks` - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay (`replicaCount: 2` and bundled PostgreSQL) and the credential-driver suite (Kubernetes Secrets plus Vault) in `Branch E2E Checks` -When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, builds the Linux VM driver artifact once, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI. VM E2E also reuses the prebuilt VM driver artifact and falls back to local VM-driver/runtime preparation for local runs or workflow invocations that omit the artifact. +When multiple labels are present, `Branch E2E Checks` builds each generic multi-architecture artifact set once and fans out enabled suites in parallel. Runtime-specific reusable workflows define the Docker, Podman, VM, and Kubernetes lanes. Composite actions own the replaceable Podman, KVM, kind, and mise setup. Each lane depends only on the artifact categories it consumes: VM does not wait for container-driver artifacts or supervisor images, and GPU does not wait for the gateway image. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E reuse matching prebuilt gateway and CLI binaries instead of compiling debug binaries in test jobs. Standalone-driver lanes additionally reuse driver-free gateway and compute-driver artifacts. Kubernetes managed-driver lanes consume published gateway and supervisor images, while the standalone-driver lane composes its gateway image from prebuilt binaries. The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while Kubernetes HA and credential-driver behavior are under active iteration: failures are visible in the workflow run but do not publish a required CI gate status. The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly. @@ -172,6 +172,10 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma |---|---| | `.github/workflows/branch-checks.yml` | Required non-E2E checks. Triggers on `push: pull-request/[0-9]+` for PR mirrors and `merge_group` for queued merges. | | `.github/workflows/branch-e2e.yml` | Standard, GPU, Kubernetes HA, and Kubernetes credential-driver E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels; merge groups run core and GPU E2E. | +| `.github/workflows/build-{cli,gateway,sandbox}-binaries.yml` | Independent target matrices used by branch and release workflows without creating skipped jobs. | +| `.github/workflows/package-release-binaries.yml` | Packages raw build artifacts into release tarballs without rebuilding them. | +| `.github/workflows/e2e-docker-test.yml`, `e2e-podman-test.yml`, `e2e-vm-test.yml`, `e2e-kubernetes-test.yml` | Reusable runtime lanes called directly by branch and release workflows. Callers select suites and declare only the artifacts each runtime consumes. | +| `.github/actions/setup-e2e-*` | Shared artifact, Podman, KVM, and kind setup used by the runtime lanes. | | `.github/workflows/helm-lint.yml` | Helm chart validation. PR mirror pushes skip lint jobs unless Helm inputs changed; merge groups always validate Helm because they represent the final integration state. | | `.github/actions/setup-nix/action.yml` | Installs Nix and configures the OpenShell Cachix cache, using read-only cache access when no authentication token is available. | | `.github/actions/pr-gate/action.yml` | Composite action that resolves PR metadata and verifies the required label is set for PR mirror pushes. Non-push events are allowed through. | diff --git a/architecture/build.md b/architecture/build.md index ff8dc424cc..d4a58165fc 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -104,7 +104,7 @@ and the supervisor image from `deploy/docker/Dockerfile.supervisor`. Neither Dockerfile compiles Rust — both copy a staged binary out of `deploy/docker/.build/prebuilt-binaries//` into the final image. -Binary staging is driven by `tasks/scripts/stage-prebuilt-binaries.sh`. Because +Local binary staging is driven by `tasks/scripts/stage-prebuilt-binaries.sh`. Because staging cross-compiles on the host, it sources `tasks/scripts/build-env.sh` and raises the per-process open-file limit before invoking `cargo zigbuild` on macOS — the static musl link opens hundreds of `.rlib` files at once and would @@ -126,9 +126,9 @@ host GNU libc target. The `glibc-static` variant uses plain `cargo build` with target architecture from `DOCKER_PLATFORM` when set. Otherwise, they require valid container engine host metadata and fail when the engine query is unavailable or reports an unsupported architecture, avoiding host-kernel -fallbacks that can target the wrong architecture. CI invokes the same staging -step via the `rust-native-build.yml` workflow (per-architecture, per-component) -and uploads the result as an artifact that the image build job downloads back +fallbacks that can target the wrong architecture. CI instead compiles binaries +in platform-specific Nix development shells through reusable workflows and the +shared `build-rust-binary` action. The image build downloads each binary artifact into the staging directory before running Buildx. Gateway and supervisor binaries staged into branch E2E, Release Dev, and Release @@ -141,17 +141,12 @@ is a different artifact from the source SBOM produced by `syft dir:.` in `tasks/sbom.toml`, which describes the checkout, and from the image SBOM attestation below, which describes a published image. -`docker-build.yml` and `rust-native-build.yml` take an `auditable` input that -defaults to false. The branch E2E gateway and supervisor image builds set it to -true so E2E exercises release-parity image binaries; other PR image builds and -standalone release artifacts stay non-auditable. The CI image gains the pinned -`cargo-auditable` tool through `mise install --locked` but ships no auditable -OpenShell binary of its own. Local staging opts in with -`OPENSHELL_AUDITABLE=1`. sccache's `RUSTC_WRAPPER` is unset only around auditable -builds, because it would otherwise wrap `cargo-auditable`'s workspace wrapper and -be misidentified as `rustc`. Auditable builds are verified by scanning the built -binary with Syft and requiring at least one decoded Cargo package; the check runs -only for those builds. +The shared binary build action compiles release artifacts with `cargo auditable`. +Branch E2E, Release Dev, and Release Tag image jobs stage those same artifacts +instead of rebuilding binaries in Docker. Each binary build scans its output with +Syft and requires at least one decoded Cargo package before uploading the +artifact. The CI image gains the pinned `cargo-auditable` tool through +`mise install --locked` but ships no auditable OpenShell binary of its own. Pushed Docker images carry minimal SLSA provenance and a per-platform SPDX SBOM generated by BuildKit's default Syft scanner. The registry exporter uses OCI diff --git a/deploy/docker/Dockerfile.gateway b/deploy/docker/Dockerfile.gateway index 62a55334ba..cf68dc967a 100644 --- a/deploy/docker/Dockerfile.gateway +++ b/deploy/docker/Dockerfile.gateway @@ -3,21 +3,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Gateway image build. -# -# The Rust binary is built natively before this image build runs and staged at: +# The GNU-linked gateway binary is staged at: # deploy/docker/.build/prebuilt-binaries//openshell-gateway # -# Use tasks/scripts/docker-build-image.sh gateway (or `mise run build:docker:gateway`) -# to stage the binary and build the image in one step. CI builds the binary -# per-architecture via the `rust-native-build.yml` workflow and uploads it as -# an artifact, which is downloaded into the same staging directory before the -# image build job runs. -# -# The runtime is distroless Debian 13, which provides glibc and the dynamic -# loader needed by the GNU-linked gateway binary while keeping the attack -# surface small. The default digest currently carries Debian glibc -# 2.41-12+deb13u3. +# Distroless Debian provides the glibc runtime required by the binary. ARG GATEWAY_BASE_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:d97bc0a941b8d4be647dc0ee75b264ddbb772f1ac5ba690a4309c00723b23775 FROM ${GATEWAY_BASE_IMAGE} AS gateway diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index c77c5c0aff..d515fd70b1 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -3,24 +3,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Supervisor image build. -# -# The final image carries the static `openshell-sandbox` binary used by Docker -# extraction, Podman image volumes, and the Kubernetes init container copy-self -# path. It also includes nftables so the Kubernetes supervisor sidecar can -# install pod-namespace egress enforcement rules. -# -# The Rust binary is built natively before this image build runs and staged at: +# The static sandbox binary is staged at: # deploy/docker/.build/prebuilt-binaries//openshell-sandbox # -# Use tasks/scripts/docker-build-image.sh supervisor (or `mise run build:docker:supervisor`) -# to stage the binary and build the image in one step. CI builds the binary -# per-architecture via the `rust-native-build.yml` workflow and uploads it as an -# artifact, which is downloaded into the same staging directory before the image -# build job runs. -# -# The binary is static under either supported libc variant (`SUPERVISOR_LIBC`: -# musl by default, or glibc-static), so this Alpine base runs it unchanged. +# Alpine supplies nftables and iptables for pod-namespace egress enforcement. FROM alpine:3.22 AS supervisor @@ -28,10 +14,8 @@ ARG TARGETARCH RUN apk add --no-cache nftables iptables iptables-legacy -# --chmod=0555 restores execute bits after the actions/upload-artifact + -# download-artifact roundtrip strips them. Ownership stays root (0:0) for -# Podman image-volume mounts, while world-execute lets the Kubernetes -# network sidecar run this binary as the dedicated non-root proxy UID. +# Keep the binary root-owned for Podman image-volume mounts and executable by +# the Kubernetes network sidecar's non-root proxy UID. COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox /openshell-sandbox ENTRYPOINT ["/openshell-sandbox"] diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 1960f83588..4acb7256bb 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -102,7 +102,7 @@ if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then echo "==> Building driver-free openshell-gateway" cargo build \ -p openshell-server --bin openshell-gateway \ - --no-default-features --features telemetry + --no-default-features else build_packages+=(-p openshell-server) fi diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index 1b2b8e4145..c512dab60a 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -220,7 +220,7 @@ e2e_build_gateway_binaries() { if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then cargo build "${jobs[@]}" \ -p openshell-server --bin openshell-gateway \ - --no-default-features --features telemetry + --no-default-features else cargo build "${jobs[@]}" \ -p openshell-server --bin openshell-gateway @@ -259,9 +259,14 @@ e2e_build_external_driver() { jobs=(-j "${CARGO_BUILD_JOBS}") fi target_dir="$(e2e_cargo_target_dir "${root}")" - printf -v "${output_var}" '%s' "${target_dir}/debug/${binary}" - echo "Building external ${binary}..." - cargo build "${jobs[@]}" -p "${package}" --bin "${binary}" + if [ -n "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" ]; then + printf -v "${output_var}" '%s' "${OPENSHELL_EXTERNAL_DRIVER_BIN}" + echo "Using prebuilt external driver at ${OPENSHELL_EXTERNAL_DRIVER_BIN}" + else + printf -v "${output_var}" '%s' "${target_dir}/debug/${binary}" + echo "Building external ${binary}..." + cargo build "${jobs[@]}" -p "${package}" --bin "${binary}" + fi if [ ! -x "${!output_var}" ]; then echo "ERROR: expected external driver binary at ${!output_var}" >&2 exit 1 diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index f83c8bafe1..08dd08113e 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -640,9 +640,15 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then echo "ERROR: external Kubernetes driver image composition currently requires a Linux build host." >&2 exit 2 fi - cargo build -p openshell-server --bin openshell-gateway \ - --no-default-features --features telemetry,bundled-z3 - cargo build -p openshell-driver-kubernetes --bin openshell-driver-kubernetes + external_gateway="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" + external_driver="${OPENSHELL_EXTERNAL_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-kubernetes}" + if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then + cargo build -p openshell-server --bin openshell-gateway \ + --no-default-features --features bundled-z3 + fi + if [ -z "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" ]; then + cargo build -p openshell-driver-kubernetes --bin openshell-driver-kubernetes + fi case "$(uname -m)" in x86_64) external_arch=amd64 ;; aarch64|arm64) external_arch=arm64 ;; @@ -650,9 +656,8 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then esac external_stage="${ROOT}/deploy/docker/.build/prebuilt-binaries/${external_arch}" mkdir -p "${external_stage}" - cp "${ROOT}/target/debug/openshell-gateway" "${external_stage}/openshell-gateway" - cp "${ROOT}/target/debug/openshell-driver-kubernetes" \ - "${external_stage}/openshell-driver-kubernetes" + cp "${external_gateway}" "${external_stage}/openshell-gateway" + cp "${external_driver}" "${external_stage}/openshell-driver-kubernetes" docker build \ --build-arg "TARGETARCH=${external_arch}" \ --build-arg "SUPERVISOR_IMAGE=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" \ diff --git a/flake.nix b/flake.nix index 68e6156e3b..e361fd1597 100644 --- a/flake.nix +++ b/flake.nix @@ -47,16 +47,20 @@ testGuestPkgs = import nixpkgs-test-guest { inherit system; }; commonDevShellPackages = with pkgs; [ actionlint + cargo-auditable cargo-deny cargo-nextest # Assemble Debian artifacts on macOS and Linux. dpkg + git # Required to find packages. pkg-config # Coverage. lcov + syft uv zizmor + zstd ]; treefmtEval = treefmt-nix.lib.evalModule pkgs { projectRootFile = "flake.nix"; @@ -65,6 +69,7 @@ rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; z3-static = pkgs.callPackage ./nix/pkgs/z3-static.nix { }; aws-lc-static = pkgs.callPackage ./nix/pkgs/aws-lc-static.nix { }; + vmRuntime = pkgs.callPackage ./nix/pkgs/vm-runtime.nix { }; testGuest = import ./nix/test-guest { inherit pkgs; qemuPkgs = testGuestPkgs; @@ -75,6 +80,8 @@ apps.test-guest = testGuest.app; apps.test-guest-cache = testGuest.cacheApp; + packages.vm-runtime = vmRuntime; + devShells = { default = (pkgs.mkShell.override { diff --git a/nix/devShells/musl.nix b/nix/devShells/musl.nix index ba7c06bf22..d0dbda93c1 100644 --- a/nix/devShells/musl.nix +++ b/nix/devShells/musl.nix @@ -9,7 +9,7 @@ let muslPkgs = pkgs.pkgsMusl; - stdenv = pkgs.stdenvAdapters.useMoldLinker muslPkgs.stdenv; + stdenv = muslPkgs.stdenv; rust-bin = rust-overlay.lib.mkRustBin { } muslPkgs; rustToolchain = (rust-bin.fromRustupToolchainFile ../../rust-toolchain.toml).override { enableLibsecret = false; diff --git a/nix/pkgs/vm-runtime.nix b/nix/pkgs/vm-runtime.nix new file mode 100644 index 0000000000..49a051e276 --- /dev/null +++ b/nix/pkgs/vm-runtime.nix @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ + fetchurl, + stdenv, + zstd, +}: + +let + runtime = + { + x86_64-linux = { + platform = "linux-x86_64"; + hash = "sha256-dw3Lc7IapCyNeE7j6dnlgd/b8Yc91/7IOi3XJORyILQ="; + }; + aarch64-linux = { + platform = "linux-aarch64"; + hash = "sha256-aJDuDb7AsuH9R+AyXA/JIxE9fJmZ5kP0Lkhg6F0Ot5A="; + }; + aarch64-darwin = { + platform = "darwin-aarch64"; + hash = "sha256-BDSeY5XGDozaBZzHTiQQX90jzsSc6shJZs5zdzludX0="; + }; + } + .${stdenv.hostPlatform.system}; + archive = fetchurl { + url = "https://github.com/NVIDIA/OpenShell/releases/download/vm-runtime/vm-runtime-${runtime.platform}.tar.zst"; + inherit (runtime) hash; + }; +in +stdenv.mkDerivation { + name = "openshell-vm-runtime-${runtime.platform}"; + + nativeBuildInputs = [ zstd ]; + dontUnpack = true; + + installPhase = '' + runHook preInstall + + mkdir -p "$out" + tar --extract --file ${archive} --directory "$out" + + runHook postInstall + ''; +} diff --git a/nix/toolchains/linux-gnu-2.28/default.nix b/nix/toolchains/linux-gnu-2.28/default.nix index 3e911f1ced..4b8b0ab453 100644 --- a/nix/toolchains/linux-gnu-2.28/default.nix +++ b/nix/toolchains/linux-gnu-2.28/default.nix @@ -31,7 +31,10 @@ let pkgs.overrideCC pkgs.stdenv ( pkgs.wrapCCWith { cc = pkgs.gccNGPackages.gcc-unwrapped.overrideAttrs (old: { - configureFlags = old.configureFlags ++ [ "--disable-fixincludes" ]; + configureFlags = old.configureFlags ++ [ + "--disable-fixincludes" + "--with-native-system-header-dir=/include" + ]; }); bintools = pkgs.wrapBintoolsWith { bintools = pkgs.binutils-unwrapped; diff --git a/scripts/baseline_workflow_metrics.py b/scripts/baseline_workflow_metrics.py index 5a15b70801..ffd5054679 100755 --- a/scripts/baseline_workflow_metrics.py +++ b/scripts/baseline_workflow_metrics.py @@ -41,9 +41,15 @@ WORKFLOWS: list[str] = [ "branch-checks.yml", "branch-e2e.yml", + "build-cli-binaries.yml", + "build-gateway-binaries.yml", + "build-sandbox-binaries.yml", "ci-image.yml", "docker-build.yml", - "e2e-test.yml", + "e2e-docker-test.yml", + "e2e-podman-test.yml", + "e2e-vm-test.yml", + "package-release-binaries.yml", "release-canary.yml", "release-dev.yml", "release-tag.yml", @@ -55,8 +61,14 @@ # workflow_run. For these, we scan all repo runs in the window and attribute # via `referenced_workflows`. REUSABLE_WORKFLOWS: set[str] = { + "build-cli-binaries.yml", + "build-gateway-binaries.yml", + "build-sandbox-binaries.yml", "docker-build.yml", - "e2e-test.yml", + "e2e-docker-test.yml", + "e2e-podman-test.yml", + "e2e-vm-test.yml", + "package-release-binaries.yml", } # Conclusions that represent a real execution on a runner. Percentile math From f68867b8691e72ba25e0fa7d9ee4f32adbc4701c Mon Sep 17 00:00:00 2001 From: krishicks Date: Thu, 27 Aug 2026 17:27:34 +0000 Subject: [PATCH 3/4] feat(gateway): identify gateways in exported traces (#2647) * feat(gateway): add installation name configuration Add a first-class operator-assigned gateway name with TOML, CLI, environment, and Helm configuration surfaces. Local gateways default to openshell, while Helm defaults to the chart fullname; operators sharing a collector across namespaces or clusters can set a globally distinct name. Signed-off-by: Kris Hicks * feat(gateway): identify gateways in exported traces Attach the configured gateway installation name and compute driver to the gateway OpenTelemetry resource so operators can filter traces from multiple installations that share a collector. Forward the gateway name and OTLP endpoint to managed external drivers so their distinct service resources carry the same installation identity. Keep service.name stable per process type, omit blank resource values, and leave per-span operation names and request attributes unchanged. Refs #2507 Signed-off-by: Kris Hicks --------- Signed-off-by: Kris Hicks --- architecture/gateway.md | 23 +++- crates/openshell-core/src/config.rs | 23 ++++ crates/openshell-driver-docker/src/main.rs | 9 +- .../src/otel_tracing.rs | 32 +++-- crates/openshell-driver-kubernetes/Cargo.toml | 2 +- crates/openshell-driver-kubernetes/README.md | 4 +- .../openshell-driver-kubernetes/src/main.rs | 14 +- .../src/otel_tracing.rs | 33 +++-- crates/openshell-driver-podman/src/main.rs | 16 ++- .../src/otel_tracing.rs | 33 +++-- crates/openshell-driver-vm/README.md | 7 +- crates/openshell-driver-vm/src/main.rs | 13 +- .../openshell-driver-vm/src/otel_tracing.rs | 34 +++-- crates/openshell-otel-test-support/src/lib.rs | 29 +++-- crates/openshell-server/src/cli.rs | 37 +++++- crates/openshell-server/src/compute/vm.rs | 18 ++- crates/openshell-server/src/config_file.rs | 5 + crates/openshell-server/src/lib.rs | 25 ++-- crates/openshell-server/src/otel_tracing.rs | 120 +++++++++++++++--- crates/openshell-server/src/tracing_setup.rs | 15 ++- deploy/helm/openshell/README.md | 1 + .../openshell/templates/gateway-config.yaml | 1 + .../openshell/tests/gateway_config_test.yaml | 16 +++ deploy/helm/openshell/values.yaml | 4 + docs/reference/gateway-config.mdx | 12 +- tasks/scripts/gateway-docker.sh | 1 + tasks/scripts/gateway-podman.sh | 3 +- tasks/scripts/gateway-vm.sh | 3 +- tasks/scripts/gateway.sh | 1 + 29 files changed, 425 insertions(+), 109 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index f86f855845..dcb1dd5414 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -673,6 +673,15 @@ Driver implementation settings live in the TOML driver tables. See `docs/reference/gateway-config.mdx` for worked per-driver examples and RFC 0003 for the full schema. +Each installation has an operator-assigned gateway name. Configure it with +`[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. +The built-in default is `openshell`; the Helm chart defaults it to the chart +fullname so every replica in one installation reports the same identity. +Operators must set a globally distinct name when one telemetry collector serves +installations in multiple Kubernetes namespaces or clusters. +The name identifies the gateway installation independently of client-side +aliases, network names, and the sandbox JWT issuer. + `database_url` is env-only and rejected when present in the file (`OPENSHELL_DB_URL` / `--db-url`). @@ -721,12 +730,14 @@ between a trace and its log lines. Store and compute-driver spans become children of the request span. Reconciliation, provider refresh, and driver-watch loops create their own operation spans because they have no inbound request to provide a parent. gRPC status is recorded when response -trailers arrive. - -The gateway forwards OTLP configuration and W3C trace context to managed -external drivers. Built-in drivers use dedicated in-process providers that -preserve the same RPC trace boundary. Each driver exports to the configured -collector under its own service name. +trailers arrive. Gateway spans carry resource attributes for the gateway +identity and configured compute driver. + +The gateway forwards OTLP configuration, its configured gateway name, and W3C +trace context to managed external drivers. Built-in drivers use dedicated +in-process providers that preserve the same RPC trace boundary. Each driver +exports to the configured collector under its own service name and carries the +gateway name as a resource attribute. Two invariants shape the failure behavior. Telemetry is diagnostic, so no OTLP failure stops the gateway from serving: a malformed endpoint is logged at diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index ce613974a4..e04e056033 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -29,6 +29,9 @@ pub const DEFAULT_SSH_PORT: u16 = 2222; /// Default gateway server port. pub const DEFAULT_SERVER_PORT: u16 = 17670; +/// Default operator-facing name for a gateway installation. +pub const DEFAULT_GATEWAY_NAME: &str = "openshell"; + /// Default container stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10; @@ -761,6 +764,9 @@ fn docker_socket_responds(path: &Path) -> bool { /// `Deserialize` impls for that purpose). #[derive(Debug, Clone)] pub struct Config { + /// Operator-assigned name for this gateway installation. + pub name: String, + /// Address to bind the server to. pub bind_address: SocketAddr, @@ -1168,6 +1174,7 @@ impl Config { /// Create a new config with optional TLS. pub fn new(tls: Option) -> Self { Self { + name: DEFAULT_GATEWAY_NAME.to_string(), bind_address: default_bind_address(), health_bind_address: None, metrics_bind_address: None, @@ -1195,6 +1202,13 @@ impl Config { } } + /// Create a new configuration with the gateway installation name. + #[must_use] + pub fn with_name(mut self, name: impl Into) -> Self { + self.name = name.into(); + self + } + /// Create a new configuration with the given bind address. #[must_use] pub const fn with_bind_address(mut self, addr: SocketAddr) -> Self { @@ -1573,6 +1587,15 @@ mod tests { assert_eq!(cfg.ttl_secs, 0); } + #[test] + fn name_defaults_and_can_be_overridden() { + assert_eq!(Config::new(None).name, "openshell"); + assert_eq!( + Config::new(None).with_name("production-us-west").name, + "production-us-west" + ); + } + #[test] fn gateway_interceptor_failure_policy_rejects_ignore() { let err = diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index 80a6353d31..2c6dbfd4c1 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -38,13 +38,18 @@ struct Args { #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] otlp_endpoint: Option, + + #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] + gateway_name: Option, } #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); - let (tracer_provider, setup_error) = - openshell_driver_docker::otel_tracing::provider_for(args.otlp_endpoint.as_deref()); + let (tracer_provider, setup_error) = openshell_driver_docker::otel_tracing::provider_for( + args.otlp_endpoint.as_deref(), + args.gateway_name.as_deref(), + ); tracing_subscriber::registry() .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level))) .with(tracing_subscriber::fmt::layer()) diff --git a/crates/openshell-driver-docker/src/otel_tracing.rs b/crates/openshell-driver-docker/src/otel_tracing.rs index f232fc1e7b..7623279c16 100644 --- a/crates/openshell-driver-docker/src/otel_tracing.rs +++ b/crates/openshell-driver-docker/src/otel_tracing.rs @@ -90,13 +90,28 @@ pub(crate) fn compute_driver_rpc_operation(path: &str) -> (&'static str, &'stati } } +/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway. #[must_use] -pub fn provider_for(endpoint: Option<&str>) -> (Option, Option) { - openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig { - endpoint, - service_name: ServiceName::Fixed(SERVICE_NAME), - service_version: Some(openshell_core::VERSION), - resource_attributes: Vec::new(), +pub fn provider_for( + endpoint: Option<&str>, + gateway_name: Option<&str>, +) -> (Option, Option) { + openshell_otel::provider_for(endpoint.map(|endpoint| { + OtlpTraceConfig { + endpoint, + service_name: ServiceName::Fixed(SERVICE_NAME), + service_version: Some(openshell_core::VERSION), + resource_attributes: gateway_name + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(|name| { + vec![opentelemetry::KeyValue::new( + "openshell.gateway.name", + name.to_string(), + )] + }) + .unwrap_or_default(), + } })) } @@ -151,11 +166,11 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn tracing_docker_driver_spans_reach_otlp_collector_with_distinct_service_name() { + async fn tracing_docker_driver_spans_reach_otlp_collector_with_resource_identity() { let _tracing_lock = super::test_lock().await; let collector = OtlpTestServer::start().await; - let (provider, error) = super::provider_for(Some(collector.endpoint())); + let (provider, error) = super::provider_for(Some(collector.endpoint()), Some("docker-dev")); assert!(error.is_none()); let provider = provider.expect("provider"); let subscriber = tracing_subscriber::registry().with(super::layer(&provider)); @@ -175,6 +190,7 @@ mod tests { .iter() .any(|span| span.name == "docker.schedule_sandbox") ); + assert_eq!(received.gateway_names, ["docker-dev"]); assert!( received .service_names diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index d7fcf19a17..6e821317c3 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -19,6 +19,7 @@ openshell-core = { path = "../openshell-core", default-features = false } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } +opentelemetry = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } prost = { workspace = true } @@ -39,7 +40,6 @@ notify = "8" [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } -opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } bytes = { workspace = true } http = { workspace = true } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index b352f6efe9..6d6e26654c 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -23,7 +23,9 @@ When the gateway configures `[openshell.gateway.otlp]`, Kubernetes compute-driver spans export to the same OTLP/gRPC collector with the service name `openshell-driver-kubernetes`. The driver preserves the gateway trace context and uses the same compute-driver RPC span names in its in-process and -standalone forms. +standalone forms. Standalone deployments set `--gateway-name` or +`OPENSHELL_GATEWAY_NAME` so exported spans carry the same +`openshell.gateway.name` resource attribute as gateway spans. When it creates an Agent Sandbox resource, the driver serializes the active W3C trace context into the controller-reserved `opentelemetry.io/trace-context` diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 712cc1cbda..7690ccee85 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -42,6 +42,9 @@ struct Args { #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] otlp_endpoint: Option, + #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] + gateway_name: Option, + #[arg(long, env = "OPENSHELL_WORKSPACE_MODE", default_value = "shared")] workspace_mode: WorkspaceMode, @@ -215,8 +218,10 @@ async fn shutdown_signal() { #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); - let (tracer_provider, setup_error) = - openshell_driver_kubernetes::otel_tracing::provider_for(args.otlp_endpoint.as_deref()); + let (tracer_provider, setup_error) = openshell_driver_kubernetes::otel_tracing::provider_for( + args.otlp_endpoint.as_deref(), + args.gateway_name.as_deref(), + ); tracing_subscriber::registry() .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level))) .with(tracing_subscriber::fmt::layer()) @@ -349,11 +354,13 @@ mod tests { use super::*; #[test] - fn accepts_gateway_otlp_endpoint() { + fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ "openshell-driver-kubernetes", "--otlp-endpoint", "http://collector.example:4317", + "--gateway-name", + "kubernetes-dev", ]) .expect("OTLP endpoint should parse"); @@ -361,5 +368,6 @@ mod tests { args.otlp_endpoint.as_deref(), Some("http://collector.example:4317") ); + assert_eq!(args.gateway_name.as_deref(), Some("kubernetes-dev")); } } diff --git a/crates/openshell-driver-kubernetes/src/otel_tracing.rs b/crates/openshell-driver-kubernetes/src/otel_tracing.rs index 1a8d40071d..02501651f8 100644 --- a/crates/openshell-driver-kubernetes/src/otel_tracing.rs +++ b/crates/openshell-driver-kubernetes/src/otel_tracing.rs @@ -12,13 +12,28 @@ const SERVICE_NAME: &str = "openshell-driver-kubernetes"; const INSTRUMENTATION_SCOPE: &str = "openshell-driver-kubernetes"; pub const IN_PROCESS_TARGET_PREFIX: &str = "openshell_driver_kubernetes"; +/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway. #[must_use] -pub fn provider_for(endpoint: Option<&str>) -> (Option, Option) { - openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig { - endpoint, - service_name: ServiceName::Fixed(SERVICE_NAME), - service_version: Some(openshell_core::VERSION), - resource_attributes: Vec::new(), +pub fn provider_for( + endpoint: Option<&str>, + gateway_name: Option<&str>, +) -> (Option, Option) { + openshell_otel::provider_for(endpoint.map(|endpoint| { + OtlpTraceConfig { + endpoint, + service_name: ServiceName::Fixed(SERVICE_NAME), + service_version: Some(openshell_core::VERSION), + resource_attributes: gateway_name + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(|name| { + vec![opentelemetry::KeyValue::new( + "openshell.gateway.name", + name.to_string(), + )] + }) + .unwrap_or_default(), + } })) } @@ -75,10 +90,11 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn tracing_kubernetes_driver_spans_reach_otlp_collector_with_distinct_service_name() { + async fn tracing_kubernetes_driver_spans_reach_otlp_collector_with_resource_identity() { let _tracing_lock = super::test_lock().await; let collector = OtlpTestServer::start().await; - let (provider, error) = super::provider_for(Some(collector.endpoint())); + let (provider, error) = + super::provider_for(Some(collector.endpoint()), Some("kubernetes-dev")); assert!(error.is_none()); let provider = provider.expect("provider"); let subscriber = tracing_subscriber::registry().with(super::layer(&provider)); @@ -98,6 +114,7 @@ mod tests { .iter() .any(|span| span.name == "kubernetes.create_sandbox") ); + assert_eq!(received.gateway_names, ["kubernetes-dev"]); assert!( received .service_names diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 7055cb089a..a0fa85d018 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -40,6 +40,9 @@ struct Args { #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] otlp_endpoint: Option, + #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] + gateway_name: Option, + /// Path to the Podman API Unix socket. #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")] podman_socket: Option, @@ -174,8 +177,10 @@ struct Args { #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); - let (tracer_provider, setup_error) = - openshell_driver_podman::otel_tracing::provider_for(args.otlp_endpoint.as_deref()); + let (tracer_provider, setup_error) = openshell_driver_podman::otel_tracing::provider_for( + args.otlp_endpoint.as_deref(), + args.gateway_name.as_deref(), + ); tracing_subscriber::registry() .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level))) .with(tracing_subscriber::fmt::layer()) @@ -306,17 +311,20 @@ mod tests { } #[test] - fn accepts_gateway_otlp_endpoint() { + fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ "openshell-driver-podman", "--otlp-endpoint", "http://collector.internal:4317", + "--gateway-name", + "production-us-west", ]) - .expect("OTLP endpoint should be accepted"); + .expect("OTLP configuration should be accepted"); assert_eq!( args.otlp_endpoint.as_deref(), Some("http://collector.internal:4317") ); + assert_eq!(args.gateway_name.as_deref(), Some("production-us-west")); } } diff --git a/crates/openshell-driver-podman/src/otel_tracing.rs b/crates/openshell-driver-podman/src/otel_tracing.rs index 00163c206e..646472c8ed 100644 --- a/crates/openshell-driver-podman/src/otel_tracing.rs +++ b/crates/openshell-driver-podman/src/otel_tracing.rs @@ -12,13 +12,28 @@ const SERVICE_NAME: &str = "openshell-driver-podman"; const INSTRUMENTATION_SCOPE: &str = "openshell-driver-podman"; pub const IN_PROCESS_TARGET_PREFIX: &str = "openshell_driver_podman"; +/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway. #[must_use] -pub fn provider_for(endpoint: Option<&str>) -> (Option, Option) { - openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig { - endpoint, - service_name: ServiceName::Fixed(SERVICE_NAME), - service_version: Some(openshell_core::VERSION), - resource_attributes: Vec::new(), +pub fn provider_for( + endpoint: Option<&str>, + gateway_name: Option<&str>, +) -> (Option, Option) { + openshell_otel::provider_for(endpoint.map(|endpoint| { + OtlpTraceConfig { + endpoint, + service_name: ServiceName::Fixed(SERVICE_NAME), + service_version: Some(openshell_core::VERSION), + resource_attributes: gateway_name + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(|name| { + vec![opentelemetry::KeyValue::new( + "openshell.gateway.name", + name.to_string(), + )] + }) + .unwrap_or_default(), + } })) } @@ -114,11 +129,12 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn podman_driver_spans_reach_otlp_collector_with_distinct_service_name() { + async fn podman_driver_spans_reach_otlp_collector_with_resource_identity() { let _tracing_lock = super::test_lock().await; let collector = OtlpTestServer::start().await; - let (provider, error) = super::provider_for(Some(collector.endpoint())); + let (provider, error) = + super::provider_for(Some(collector.endpoint()), Some("production-us-west")); assert!(error.is_none()); let provider = provider.expect("provider"); let subscriber = tracing_subscriber::registry().with(super::layer(&provider)); @@ -138,6 +154,7 @@ mod tests { .iter() .any(|span| span.name == "podman.create") ); + assert_eq!(received.gateway_names, ["production-us-west"]); assert!( received .service_names diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 19ac66c3f9..67666be735 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -40,7 +40,10 @@ First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw By default `mise run gateway:vm`: - Listens on plaintext HTTP at `127.0.0.1:18081`. -- Registers the CLI gateway `vm-dev` by writing `~/.config/openshell/gateways/vm-dev/metadata.json`. It does not modify the workspace `.env`. +- Configures the gateway installation name as `vm-dev` and registers the same + name with the CLI by writing + `~/.config/openshell/gateways/vm-dev/metadata.json`. It does not modify the + workspace `.env`. - Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`. - Places the VM driver state (per-sandbox `overlay.ext4`, image cache, and `run/compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`. - Writes `.cache/gateway-vm/gateway.toml` with `[openshell.drivers.vm].driver_dir = "$PWD/target/debug"` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`. @@ -68,7 +71,7 @@ Override defaults via environment: # custom port (fails fast if in use) OPENSHELL_SERVER_PORT=18091 mise run gateway:vm -# custom CLI gateway name + namespace +# custom gateway installation/CLI name + namespace OPENSHELL_VM_GATEWAY_NAME=vm-feature-a \ OPENSHELL_SANDBOX_NAMESPACE=vm-feature-a \ mise run gateway:vm diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 949d4ce05c..95ebf0f8b2 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -91,6 +91,9 @@ struct Args { #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] otlp_endpoint: Option, + #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] + gateway_name: Option, + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] openshell_endpoint: Option, @@ -186,8 +189,10 @@ async fn main() -> Result<()> { return Ok(()); } - let (tracer_provider, setup_error) = - openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref()); + let (tracer_provider, setup_error) = openshell_driver_vm::otel_tracing::provider_for( + args.otlp_endpoint.as_deref(), + args.gateway_name.as_deref(), + ); tracing_subscriber::registry() .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level))) .with(tracing_subscriber::fmt::layer()) @@ -691,11 +696,13 @@ mod tests { } #[test] - fn accepts_gateway_otlp_endpoint() { + fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ "openshell-driver-vm", "--otlp-endpoint", "http://127.0.0.1:4317", + "--gateway-name", + "production-us-west", ]); assert!( args.is_ok(), diff --git a/crates/openshell-driver-vm/src/otel_tracing.rs b/crates/openshell-driver-vm/src/otel_tracing.rs index f63066bb88..47e2dd5f73 100644 --- a/crates/openshell-driver-vm/src/otel_tracing.rs +++ b/crates/openshell-driver-vm/src/otel_tracing.rs @@ -11,14 +11,28 @@ use tracing_subscriber::registry::LookupSpan; const SERVICE_NAME: &str = "openshell-driver-vm"; const INSTRUMENTATION_SCOPE: &str = "openshell-driver-vm"; -/// Build a tracer provider for the configured OTLP/gRPC endpoint. +/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway. #[must_use] -pub fn provider_for(endpoint: Option<&str>) -> (Option, Option) { - openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig { - endpoint, - service_name: ServiceName::Fixed(SERVICE_NAME), - service_version: Some(openshell_core::VERSION), - resource_attributes: Vec::new(), +pub fn provider_for( + endpoint: Option<&str>, + gateway_name: Option<&str>, +) -> (Option, Option) { + openshell_otel::provider_for(endpoint.map(|endpoint| { + OtlpTraceConfig { + endpoint, + service_name: ServiceName::Fixed(SERVICE_NAME), + service_version: Some(openshell_core::VERSION), + resource_attributes: gateway_name + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(|name| { + vec![opentelemetry::KeyValue::new( + "openshell.gateway.name", + name.to_string(), + )] + }) + .unwrap_or_default(), + } })) } @@ -83,10 +97,11 @@ mod tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn vm_driver_spans_reach_otlp_collector_with_distinct_service_name() { + async fn vm_driver_spans_reach_otlp_collector_with_resource_identity() { let collector = OtlpTestServer::start().await; - let (provider, error) = super::provider_for(Some(collector.endpoint())); + let (provider, error) = + super::provider_for(Some(collector.endpoint()), Some("production-us-west")); assert!(error.is_none(), "valid OTLP endpoint should configure"); let provider = provider.expect("provider"); let subscriber = tracing_subscriber::registry().with(super::layer(&provider)); @@ -113,5 +128,6 @@ mod tests { "VM spans should use a distinct service name, got {:?}", received.service_names ); + assert_eq!(received.gateway_names, ["production-us-west"]); } } diff --git a/crates/openshell-otel-test-support/src/lib.rs b/crates/openshell-otel-test-support/src/lib.rs index 3f40d4b74a..4edbe8b7cc 100644 --- a/crates/openshell-otel-test-support/src/lib.rs +++ b/crates/openshell-otel-test-support/src/lib.rs @@ -16,6 +16,7 @@ use opentelemetry_proto::tonic::trace::v1::Span; pub struct ReceivedTraces { pub spans: Vec, pub service_names: Vec, + pub gateway_names: Vec, } #[derive(Clone)] @@ -37,18 +38,22 @@ impl TraceService for Collector { .unwrap_or_else(std::sync::PoisonError::into_inner); for resource_span in request.into_inner().resource_spans { if let Some(resource) = resource_span.resource { - received.service_names.extend( - resource - .attributes - .into_iter() - .filter(|attribute| attribute.key == "service.name") - .filter_map(|attribute| attribute.value) - .filter_map(|value| value.value) - .filter_map(|value| match value { - opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value), - _ => None, - }), - ); + for attribute in resource.attributes { + let Some(value) = attribute.value.and_then(|value| value.value) else { + continue; + }; + let opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( + value, + ) = value + else { + continue; + }; + match attribute.key.as_str() { + "service.name" => received.service_names.push(value), + "openshell.gateway.name" => received.gateway_names.push(value), + _ => {} + } + } } for scope_span in resource_span.scope_spans { received.spans.extend(scope_span.spans); diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 1ca68895f0..f22e1355e4 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -7,7 +7,7 @@ use clap::parser::ValueSource; use clap::{ArgAction, ArgMatches, Command, CommandFactory, FromArgMatches, Parser}; use miette::{IntoDiagnostic, Result}; use openshell_core::ComputeDriverKind; -use openshell_core::config::DEFAULT_SERVER_PORT; +use openshell_core::config::{DEFAULT_GATEWAY_NAME, DEFAULT_SERVER_PORT}; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use tracing::{error, info, warn}; @@ -55,6 +55,14 @@ struct RunArgs { #[arg(long, env = "OPENSHELL_GATEWAY_CONFIG")] config: Option, + /// Operator-assigned name for this gateway installation. + #[arg( + long = "name", + default_value = DEFAULT_GATEWAY_NAME, + env = "OPENSHELL_GATEWAY_NAME" + )] + name: String, + /// IP address to bind the server, health, and metrics listeners to. #[arg(long, default_value = "127.0.0.1", env = "OPENSHELL_BIND_ADDRESS")] bind_address: IpAddr, @@ -336,7 +344,13 @@ fn prepare_server_config( .clone() .expect("runtime defaults populate db_url"); + let name = args.name.trim(); + if name.is_empty() { + return Err(miette::miette!("gateway name must not be empty")); + } + let mut config = openshell_core::Config::new(tls) + .with_name(name) .with_bind_address(bind) .with_log_level(&args.log_level); if let Some(auth) = file.as_ref().and_then(|f| f.openshell.gateway.auth.clone()) { @@ -495,12 +509,17 @@ async fn run_from_args( .config_file .as_ref() .and_then(|f| f.openshell.gateway.otlp.as_ref()); + let gateway_resource = crate::otel_tracing::GatewayResourceAttributes::new( + Some(prepared.config.name.as_str()), + Some(compute_driver.name()), + ); let (tracing_handle, setup_error) = crate::tracing_setup::install( EnvFilter::try_from_default_env() .unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)), &tracing_log_bus, otlp_config, &compute_driver, + gateway_resource, ); let has_client_ca = prepared @@ -651,6 +670,11 @@ fn resolve_aux_listener( /// The function intentionally does not touch `database_url` — that secret is /// env-only and the loader already rejected it when it appears in the file. fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: &ArgMatches) { + if let Some(name) = &file.name + && arg_defaulted(matches, "name") + { + args.name.clone_from(name); + } if let Some(addr) = file.bind_address { if arg_defaulted(matches, "bind_address") { args.bind_address = addr.ip(); @@ -1452,12 +1476,14 @@ enabled = false let _g1 = EnvVarGuard::remove("OPENSHELL_BIND_ADDRESS"); let _g2 = EnvVarGuard::remove("OPENSHELL_SERVER_PORT"); let _g3 = EnvVarGuard::remove("OPENSHELL_LOG_LEVEL"); + let _g4 = EnvVarGuard::remove("OPENSHELL_GATEWAY_NAME"); let (mut args, matches) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); let file = config_file_from_toml( r#" [openshell.gateway] +name = "production-us-west" bind_address = "0.0.0.0:9090" log_level = "debug" "#, @@ -1467,6 +1493,7 @@ log_level = "debug" assert_eq!(args.bind_address, IpAddr::V4(Ipv4Addr::UNSPECIFIED)); assert_eq!(args.port, 9090); assert_eq!(args.log_level, "debug"); + assert_eq!(args.name, "production-us-west"); } #[test] @@ -1476,6 +1503,7 @@ log_level = "debug" .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_BIND_ADDRESS"); let _g2 = EnvVarGuard::remove("OPENSHELL_LOG_LEVEL"); + let _g3 = EnvVarGuard::remove("OPENSHELL_GATEWAY_NAME"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", @@ -1483,16 +1511,20 @@ log_level = "debug" "sqlite::memory:", "--log-level", "warn", + "--name", + "cli-gateway", ]); let file = config_file_from_toml( r#" [openshell.gateway] +name = "file-gateway" log_level = "debug" "#, ); merge_file_into_args(&mut args, &file.openshell.gateway, &matches); assert_eq!(args.log_level, "warn", "CLI flag must win over file"); + assert_eq!(args.name, "cli-gateway"); } #[test] @@ -1501,18 +1533,21 @@ log_level = "debug" .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g = EnvVarGuard::set("OPENSHELL_LOG_LEVEL", "trace"); + let _g2 = EnvVarGuard::set("OPENSHELL_GATEWAY_NAME", "env-gateway"); let (mut args, matches) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); let file = config_file_from_toml( r#" [openshell.gateway] +name = "file-gateway" log_level = "debug" "#, ); merge_file_into_args(&mut args, &file.openshell.gateway, &matches); assert_eq!(args.log_level, "trace", "env var must win over file"); + assert_eq!(args.name, "env-gateway"); } #[test] diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs index 80b445d201..6a66fc8aa5 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-server/src/compute/vm.rs @@ -478,7 +478,7 @@ pub async fn spawn( .arg("--expected-peer-pid") .arg(std::process::id().to_string()); command.arg("--log-level").arg(&config.log_level); - append_otlp_args(&mut command, otlp_config); + append_otlp_args(&mut command, otlp_config, &config.name); command .arg("--openshell-endpoint") .arg(&vm_config.grpc_endpoint); @@ -521,9 +521,10 @@ pub async fn spawn( } #[cfg(unix)] -fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>) { +fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) { if let Some(config) = otlp_config { command.arg("--otlp-endpoint").arg(&config.endpoint); + command.arg("--gateway-name").arg(gateway_name); } } @@ -621,7 +622,7 @@ mod tests { use tempfile::tempdir; #[test] - fn vm_driver_command_includes_gateway_otlp_endpoint() { + fn vm_driver_command_includes_gateway_otlp_configuration() { let mut command = tokio::process::Command::new("openshell-driver-vm"); append_otlp_args( &mut command, @@ -629,6 +630,7 @@ mod tests { endpoint: "http://collector.internal:4317".to_string(), service_name: Some("custom-gateway".to_string()), }), + "production-us-west", ); let args = command @@ -636,7 +638,15 @@ mod tests { .get_args() .map(|arg| arg.to_string_lossy().into_owned()) .collect::>(); - assert_eq!(args, ["--otlp-endpoint", "http://collector.internal:4317"]); + assert_eq!( + args, + [ + "--otlp-endpoint", + "http://collector.internal:4317", + "--gateway-name", + "production-us-west" + ] + ); } #[tokio::test] diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index f960a45d8d..74b6aad01b 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -87,6 +87,11 @@ pub struct OpenShellRoot { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct GatewayFileSection { + // ── Identity ───────────────────────────────────────────────────────── + /// Operator-assigned name for this gateway installation. + #[serde(default)] + pub name: Option, + // ── Listeners ──────────────────────────────────────────────────────── #[serde(default)] pub bind_address: Option, diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index cad48ebc3c..093be3752d 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -583,13 +583,8 @@ pub(crate) async fn run_server( let sandbox_index = SandboxIndex::new(); let sandbox_watch_bus = SandboxWatchBus::new(); let supervisor_sessions = Arc::new(supervisor_session::SupervisorSessionRegistry::new()); - let driver_startup = compute::driver_config::DriverStartupContext { - file: config_file.as_ref(), - guest_tls: guest_tls.as_ref(), - gateway_port: config.bind_address.port(), - gateway_tls_enabled: config.tls.is_some(), - endpoint_overrides: &config.compute_driver_endpoints, - }; + let driver_startup = + compute_driver_startup_context(&config, config_file.as_ref(), guest_tls.as_ref()); let (compute, operator_allowlist) = build_compute_runtime( &config, driver_startup, @@ -1668,6 +1663,20 @@ async fn build_compute_runtime( Ok((runtime, operator_allowlist)) } +fn compute_driver_startup_context<'a>( + config: &'a Config, + config_file: Option<&'a config_file::ConfigFile>, + guest_tls: Option<&'a compute::driver_config::GuestTlsPaths>, +) -> compute::driver_config::DriverStartupContext<'a> { + compute::driver_config::DriverStartupContext { + file: config_file, + guest_tls, + gateway_port: config.bind_address.port(), + gateway_tls_enabled: config.tls.is_some(), + endpoint_overrides: &config.compute_driver_endpoints, + } +} + #[derive(Debug, Clone)] pub(crate) enum ConfiguredComputeDriver { Registered(ComputeDriverRegistration), @@ -1675,7 +1684,7 @@ pub(crate) enum ConfiguredComputeDriver { } impl ConfiguredComputeDriver { - fn name(&self) -> &str { + pub(crate) fn name(&self) -> &str { match self { Self::Registered(registration) => ®istration.name, Self::Remote { name } => name, diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index 6a88b2f564..b4e8d9e506 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -37,7 +37,31 @@ const DEFAULT_SERVICE_NAME: &str = "openshell-gateway"; /// Instrumentation scope recorded on spans this gateway emits. const INSTRUMENTATION_SCOPE: &str = "openshell-gateway"; -fn trace_config(cfg: &OtlpConfig) -> OtlpTraceConfig<'_> { +/// Gateway identity recorded on every exported span. +#[derive(Debug, Clone, Copy, Default)] +pub struct GatewayResourceAttributes<'a> { + name: Option<&'a str>, + compute_driver: Option<&'a str>, +} + +impl<'a> GatewayResourceAttributes<'a> { + pub fn new(name: Option<&'a str>, compute_driver: Option<&'a str>) -> Self { + Self { + name, + compute_driver, + } + } + + /// The configured gateway installation name, if any. + pub fn name(&self) -> Option<&'a str> { + self.name + } +} + +fn trace_config<'cfg>( + cfg: &'cfg OtlpConfig, + gateway: GatewayResourceAttributes<'_>, +) -> OtlpTraceConfig<'cfg> { let service_name = cfg .service_name .as_deref() @@ -48,17 +72,35 @@ fn trace_config(cfg: &OtlpConfig) -> OtlpTraceConfig<'_> { ServiceName::Fixed, ); + let mut resource_attributes = Vec::new(); + if let Some(name) = gateway.name.map(str::trim).filter(|s| !s.is_empty()) { + resource_attributes.push(opentelemetry::KeyValue::new( + "openshell.gateway.name", + name.to_string(), + )); + } + if let Some(compute_driver) = gateway + .compute_driver + .map(str::trim) + .filter(|s| !s.is_empty()) + { + resource_attributes.push(opentelemetry::KeyValue::new( + "openshell.gateway.compute_driver", + compute_driver.to_string(), + )); + } + OtlpTraceConfig { endpoint: &cfg.endpoint, service_name, service_version: Some(openshell_core::VERSION), - resource_attributes: Vec::new(), + resource_attributes, } } #[cfg(test)] -fn build_resource(cfg: &OtlpConfig) -> Resource { - openshell_otel::resource_for(&trace_config(cfg)) +fn build_resource(cfg: &OtlpConfig, gateway: GatewayResourceAttributes<'_>) -> Resource { + openshell_otel::resource_for(&trace_config(cfg, gateway)) } /// Build a tracer provider exporting over OTLP/gRPC to the configured endpoint. @@ -70,8 +112,11 @@ fn build_resource(cfg: &OtlpConfig) -> Resource { /// The sampler and span limits are left at the SDK's defaults, which are /// themselves resolved from `OTEL_*` env vars (see the module docs). #[cfg(test)] -fn build_provider(cfg: &OtlpConfig) -> Result { - openshell_otel::build_provider(&trace_config(cfg)) +fn build_provider( + cfg: &OtlpConfig, + gateway: GatewayResourceAttributes<'_>, +) -> Result { + openshell_otel::build_provider(&trace_config(cfg, gateway)) } /// Resolve the tracer provider for a gateway config file's optional @@ -82,8 +127,11 @@ fn build_provider(cfg: &OtlpConfig) -> Result { /// /// The error is returned rather than logged because the provider is built /// before the subscriber it attaches to, so logging here would go nowhere. -pub fn provider_for(cfg: Option<&OtlpConfig>) -> (Option, Option) { - openshell_otel::provider_for(cfg.map(trace_config)) +pub fn provider_for( + cfg: Option<&OtlpConfig>, + gateway: GatewayResourceAttributes<'_>, +) -> (Option, Option) { + openshell_otel::provider_for(cfg.map(|cfg| trace_config(cfg, gateway))) } /// Build the gateway layer while routing one selected in-process driver to @@ -259,6 +307,10 @@ mod tests { } } + fn build_test_resource(cfg: &OtlpConfig) -> Resource { + build_resource(cfg, GatewayResourceAttributes::default()) + } + #[test] fn resource_defaults_the_service_name() { let _lock = crate::TEST_ENV_LOCK @@ -267,7 +319,7 @@ mod tests { let _env = EnvVarGuard::remove("OTEL_SERVICE_NAME"); assert_eq!( - service_name_of(&build_resource(&config())), + service_name_of(&build_test_resource(&config())), Some(DEFAULT_SERVICE_NAME.to_string()) ); } @@ -276,7 +328,7 @@ mod tests { fn resource_honors_configured_service_name_and_carries_version() { let mut cfg = config(); cfg.service_name = Some("gateway-staging".into()); - let resource = build_resource(&cfg); + let resource = build_test_resource(&cfg); assert_eq!( resource @@ -292,6 +344,31 @@ mod tests { ); } + #[test] + fn resource_carries_gateway_name_and_compute_driver() { + let resource = build_resource( + &config(), + GatewayResourceAttributes::new(Some("vm-dev"), Some("vm")), + ); + + assert_eq!( + resource + .get(&opentelemetry::Key::from_static_str( + "openshell.gateway.name", + )) + .map(|v| v.to_string()), + Some("vm-dev".to_string()) + ); + assert_eq!( + resource + .get(&opentelemetry::Key::from_static_str( + "openshell.gateway.compute_driver", + )) + .map(|v| v.to_string()), + Some("vm".to_string()) + ); + } + struct EnvVarGuard { key: &'static str, original: Option, @@ -346,7 +423,7 @@ mod tests { cfg.service_name = Some("from-config".into()); assert_eq!( - service_name_of(&build_resource(&cfg)), + service_name_of(&build_test_resource(&cfg)), Some("from-config".to_string()) ); } @@ -361,7 +438,7 @@ mod tests { let _env = EnvVarGuard::set("OTEL_SERVICE_NAME", "from-env"); assert_eq!( - service_name_of(&build_resource(&config())), + service_name_of(&build_test_resource(&config())), Some("from-env".to_string()) ); } @@ -376,7 +453,7 @@ mod tests { let mut cfg = config(); cfg.service_name = Some(" ".into()); assert_eq!( - service_name_of(&build_resource(&cfg)), + service_name_of(&build_test_resource(&cfg)), Some(DEFAULT_SERVICE_NAME.to_string()) ); } @@ -385,7 +462,8 @@ mod tests { fn provider_rejects_a_malformed_endpoint() { let mut cfg = config(); cfg.endpoint = "definitely not a url".into(); - let err = build_provider(&cfg).expect_err("malformed endpoint"); + let err = build_provider(&cfg, GatewayResourceAttributes::default()) + .expect_err("malformed endpoint"); assert!( err.to_string().contains("definitely not a url"), "error names the offending endpoint: {err}" @@ -396,7 +474,10 @@ mod tests { fn provider_rejects_an_empty_endpoint() { let mut cfg = config(); cfg.endpoint = " ".into(); - assert!(build_provider(&cfg).is_err(), "empty endpoint is rejected"); + assert!( + build_provider(&cfg, GatewayResourceAttributes::default()).is_err(), + "empty endpoint is rejected" + ); } #[tokio::test] @@ -404,7 +485,8 @@ mod tests { // The OTLP batch exporter connects lazily, so a valid endpoint must // build even when nothing is listening — the gateway must not fail to // start because its collector is down. - let provider = build_provider(&config()).expect("provider builds"); + let provider = build_provider(&config(), GatewayResourceAttributes::default()) + .expect("provider builds"); provider.shutdown().ok(); } @@ -413,7 +495,7 @@ mod tests { /// error for the caller to log — see the misconfigured-endpoint test. #[tokio::test] async fn absent_otlp_table_disables_export() { - let (provider, err) = provider_for(None); + let (provider, err) = provider_for(None, GatewayResourceAttributes::default()); assert!(provider.is_none(), "export is off"); assert!( err.is_none(), @@ -423,7 +505,7 @@ mod tests { #[tokio::test] async fn present_otlp_table_enables_export() { - let (provider, err) = provider_for(Some(&config())); + let (provider, err) = provider_for(Some(&config()), GatewayResourceAttributes::default()); assert!(err.is_none()); provider.expect("provider is present").shutdown().ok(); } @@ -436,7 +518,7 @@ mod tests { let mut cfg = config(); cfg.endpoint = "definitely not a url".into(); - let (provider, err) = provider_for(Some(&cfg)); + let (provider, err) = provider_for(Some(&cfg), GatewayResourceAttributes::default()); assert!( provider.is_none(), "a bad endpoint degrades to no export rather than failing startup" diff --git a/crates/openshell-server/src/tracing_setup.rs b/crates/openshell-server/src/tracing_setup.rs index 968a1c6a05..edcf303072 100644 --- a/crates/openshell-server/src/tracing_setup.rs +++ b/crates/openshell-server/src/tracing_setup.rs @@ -13,7 +13,7 @@ use tracing_subscriber::prelude::*; use crate::ConfiguredComputeDriver; use crate::config_file::OtlpConfig; -use crate::otel_tracing::SetupError; +use crate::otel_tracing::{GatewayResourceAttributes, SetupError}; use crate::tracing_bus::TracingLogBus; pub struct TracingHandle { @@ -94,16 +94,17 @@ fn in_process_driver_target_prefix(driver: Option) -> Op fn in_process_driver_provider( driver: Option, endpoint: Option<&str>, + gateway_name: Option<&str>, ) -> (Option, Option) { match driver { Some(InProcessDriverTracing::Docker) => { - openshell_driver_docker::otel_tracing::provider_for(endpoint) + openshell_driver_docker::otel_tracing::provider_for(endpoint, gateway_name) } Some(InProcessDriverTracing::Kubernetes) => { - openshell_driver_kubernetes::otel_tracing::provider_for(endpoint) + openshell_driver_kubernetes::otel_tracing::provider_for(endpoint, gateway_name) } Some(InProcessDriverTracing::Podman) => { - openshell_driver_podman::otel_tracing::provider_for(endpoint) + openshell_driver_podman::otel_tracing::provider_for(endpoint, gateway_name) } None => (None, None), } @@ -113,6 +114,7 @@ fn in_process_driver_provider( fn in_process_driver_provider( _driver: Option, _endpoint: Option<&str>, + _gateway_name: Option<&str>, ) -> (Option, Option) { (None, None) } @@ -155,8 +157,9 @@ pub fn install( tracing_log_bus: &TracingLogBus, otlp_config: Option<&OtlpConfig>, driver: &ConfiguredComputeDriver, + gateway: GatewayResourceAttributes<'_>, ) -> (TracingHandle, Option) { - let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config); + let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config, gateway); let selected_driver = in_process_driver_tracing(driver); let driver_endpoint = selected_driver .is_some() @@ -164,7 +167,7 @@ pub fn install( .flatten() .map(|config| config.endpoint.as_str()); let (driver_tracer_provider, driver_setup_error) = - in_process_driver_provider(selected_driver, driver_endpoint); + in_process_driver_provider(selected_driver, driver_endpoint, gateway.name()); tracing_subscriber::registry() .with(env_filter) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 25550f3cab..1786a46f63 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -253,6 +253,7 @@ discovery endpoint or its TLS CA. | server.grpcRateLimit.windowSeconds | int | `0` | gRPC rate-limit window length in seconds. Must be positive (alongside requests) to enable rate limiting; 0 (default) disables it. | | server.hostGatewayIP | string | `""` | Host gateway IP for sandbox pod hostAliases. When set, sandbox pods get hostAliases entries mapping host.docker.internal and host.openshell.internal to this IP, allowing them to reach services running on the Docker host. Auto-detected by the cluster entrypoint script. | | server.logLevel | string | `"info"` | Gateway log level. | +| server.name | string | `""` | Operator-facing gateway name. Defaults to the chart fullname so all replicas in one installation share an identity. Set explicitly when one telemetry collector receives spans from multiple namespaces or clusters. | | server.oidc.adminRole | string | `""` | Role name for admin access. Leave empty (with userRole also empty) for authentication-only mode. Both must be set or both empty. | | server.oidc.audience | string | `"openshell-cli"` | Expected audience claim for the API resource server. This should match the server's --oidc-audience, NOT the CLI client ID. | | server.oidc.caConfigMapName | string | `""` | Name of a ConfigMap containing a CA certificate bundle (key: ca.crt) for verifying the OIDC issuer's TLS certificate. Required when the issuer uses a non-public CA (e.g. OpenShift ingress, private PKI). | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 60420cc535..d7f3cb9a83 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -35,6 +35,7 @@ data: version = 1 [openshell.gateway] + name = {{ .Values.server.name | default (include "openshell.fullname" .) | quote }} bind_address = "0.0.0.0:{{ .Values.service.port }}" {{- if .Values.service.healthPort }} health_bind_address = "0.0.0.0:{{ .Values.service.healthPort }}" diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 10e14d6554..b54b036c0f 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -11,6 +11,22 @@ release: namespace: my-namespace tests: + - it: identifies the gateway by chart fullname by default + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^name\s*=\s*"openshell"$' + + - it: renders an explicit gateway name + template: templates/gateway-config.yaml + set: + server.name: production-us-west + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^name\s*=\s*"production-us-west"$' + # Regression for Drew's P2: a ConfigMap-only mutation in `helm upgrade` # must roll the StatefulSet, otherwise pods keep running with stale config. - it: annotates the StatefulSet pod template with a ConfigMap checksum diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 6c0889773c..6b0b6242b0 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -189,6 +189,10 @@ affinity: {} # Server configuration server: + # -- Operator-facing gateway name. Defaults to the chart fullname so all + # replicas in one installation share an identity. Set explicitly when one + # telemetry collector receives spans from multiple namespaces or clusters. + name: "" # -- Gateway log level. logLevel: info # OpenTelemetry trace export over OTLP/gRPC. Leave endpoint empty to disable. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index a066b14f01..32d93aa2e6 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -18,6 +18,8 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa `database_url` is env-only. The loader rejects it when it appears in the file. When `OPENSHELL_DB_URL` is unset, the gateway stores its SQLite database under `$XDG_STATE_HOME/openshell/gateway/openshell.db`. +`name` assigns an operator-facing identity to the gateway installation. Set it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. It defaults to `openshell`; the Helm chart defaults it to the chart fullname so all replicas in one installation share a name. Chart fullnames are only unique within their Kubernetes namespace, so set `server.name` explicitly when one collector receives telemetry from multiple namespaces or clusters. This identity is independent of client-side gateway aliases, TLS names, and `gateway_jwt.gateway_id`. + ## Package-Managed Locations Package-managed gateways do not require a TOML file. Create one at the package's optional config location when you need to override built-in defaults. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. @@ -69,6 +71,7 @@ A complete gateway configuration covering every section. Trim to the fields you version = 1 [openshell.gateway] +name = "production-us-west" bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" metrics_bind_address = "0.0.0.0:9090" @@ -228,11 +231,11 @@ The transport is **OTLP over gRPC only**. HTTP/protobuf and HTTP/JSON are not su The OpenTelemetry SDK logs export failures after startup. Spans in a failed batch are dropped rather than retried. -`service_name` sets the gateway's `service.name` resource attribute and defaults to `openshell-gateway`. The gateway also reports `service.version`. +`service_name` sets the gateway's `service.name` resource attribute and defaults to `openshell-gateway`. The gateway also reports `service.version`, `openshell.gateway.name` from the gateway's configured `name`, and `openshell.gateway.compute_driver`. Only OpenTelemetry traces are exported. Inbound gRPC and HTTP requests produce server spans named for the RPC or HTTP method. Store and compute-driver operations appear as child spans. Internal reconciliation, credential-refresh, and driver-watch loops create operation roots for their store work because no inbound request supplies a parent. The gateway continues valid W3C `traceparent` context and starts a new trace when none is supplied. Request spans carry `method`, `path`, and the `request_id` that also appears in gateway logs. Health endpoint spans use DEBUG level and are not exported by the default INFO filter. -The gateway forwards the OTLP configuration and W3C trace context to managed external drivers. Built-in Docker and Podman drivers also export their spans to the same collector through dedicated in-process providers. Driver spans retain the gateway trace context and use a distinct service name such as `openshell-driver-docker` or `openshell-driver-podman`. +The gateway forwards the OTLP configuration, configured gateway name, and W3C trace context to managed external drivers. Built-in drivers also export their spans to the same collector through dedicated in-process providers. Driver spans retain the gateway trace context, use a distinct service name such as `openshell-driver-docker` or `openshell-driver-podman`, and carry the gateway name as the `openshell.gateway.name` resource attribute. Operator-run external drivers own their own telemetry configuration. For Helm deployments, set `server.otlp.endpoint` to render this table. The optional `server.otlp.serviceName` value overrides the gateway service name; @@ -242,6 +245,11 @@ The local `mise run helm:k3s:create` workflow installs a trace collector and UI, enables Agent Sandbox controller tracing on supported releases, and configures both the controller and the existing Skaffold deployment to export traces to it automatically. +The local `gateway`, `gateway:docker`, `gateway:podman`, and `gateway:vm` tasks +also set the gateway installation name in their generated configuration. Their +defaults are driver-specific (`kubernetes-dev`, `docker-dev`, `podman-dev`, and +`vm-dev`), and their documented `OPENSHELL_*_GATEWAY_NAME` overrides update both +CLI registration and exported telemetry identity. Run `mise run helm:k3s:forward` to forward OTLP/gRPC to local port `4317` and the trace UI to local port `18888`. When Skaffold has deployed a Kubernetes gateway, the task also forwards it to local port `8090`; otherwise it continues diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 6e7b85634a..a1f9e93e6a 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -199,6 +199,7 @@ cat >"${CONFIG_PATH}" <"${CONFIG_PATH}" <- so the AF_UNIX socket @@ -324,6 +324,7 @@ cat >"${CONFIG_PATH}" <"${CONFIG_PATH}" < Date: Thu, 27 Aug 2026 17:55:01 +0000 Subject: [PATCH 4/4] fix(dev): harden local cluster and gateway startup (#2993) * fix(helm): refresh kubeconfig for existing k3d clusters Docker can recreate the k3d load balancer on a new API port. Start existing clusters and prefer fresh k3d entries so create does not retain a stale endpoint. Signed-off-by: Kris Hicks * fix(dev): conditionally enable local OTLP export Probe port 4317 before adding OTLP configuration for the VM, Docker, and Podman gateway tasks. Document the startup behavior and troubleshooting for local collector availability. Signed-off-by: Kris Hicks --------- Signed-off-by: Kris Hicks --- .../skills/debug-openshell-cluster/SKILL.md | 1 + .agents/skills/helm-dev-environment/SKILL.md | 7 +++++-- crates/openshell-driver-docker/README.md | 4 ++++ crates/openshell-driver-podman/README.md | 4 ++++ crates/openshell-driver-vm/README.md | 1 + tasks/scripts/gateway-docker.sh | 20 ++++++++++++++++--- tasks/scripts/gateway-podman.sh | 20 ++++++++++++++++--- tasks/scripts/gateway-vm.sh | 20 ++++++++++++++++--- tasks/scripts/helm-k3s-local.sh | 10 +++++++--- 9 files changed, 73 insertions(+), 14 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 80d5057b17..90b9a38409 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -600,6 +600,7 @@ openshell logs | Symptom | Likely cause | Check | |---|---|---| | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | +| `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | | Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | | Admin, health, reflection, or HTTP request is denied on an additional Docker/Podman callback-only listener | Additional callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected | diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index ebd9a01595..c60733e954 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -26,6 +26,8 @@ mise run helm:k3s:create ``` Creates a k3d cluster and merges its kubeconfig into the worktree-local `kubeconfig` file. +When the named cluster already exists, the task starts any stopped containers and refreshes +same-named kubeconfig entries so a recreated load balancer's current API port takes effect. Also applies the upstream agent-sandbox CRDs/controller (pinned via `AGENT_SANDBOX_VERSION` in `tasks/scripts/helm-k3s-local.sh`, fetched from `github.com/kubernetes-sigs/agent-sandbox` releases), enables its OTLP tracing on v0.5 and later, installs an OTLP trace @@ -118,8 +120,9 @@ Kubernetes compute-driver spans under their distinct service names, along with Agent Sandbox controller reconciliation spans linked through the Sandbox trace-context annotation. The same command exposes OTLP/gRPC on `http://127.0.0.1:4317` and, when deployed, the Kubernetes gateway on -`http://127.0.0.1:8090`; the local `gateway`, `gateway:docker`, and `gateway:vm` -tasks export to the collector endpoint automatically. +`http://127.0.0.1:8090`. The local `gateway:docker`, `gateway:podman`, and +`gateway:vm` tasks detect the collector listener at startup and enable trace +export only while it is reachable. **HA test deploy** (two gateway replicas + external PostgreSQL Secret): uncomment `#- ci/values-high-availability.yaml` in `deploy/helm/openshell/skaffold.yaml`, diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 9b9657e94d..bbd7e69b88 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -8,6 +8,10 @@ spans export to the same OTLP/gRPC collector with the service name context and emits the compute-driver RPC boundary that a standalone driver would expose. +`mise run gateway:docker` enables this export only when a local collector is +listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration +so the development gateway does not repeatedly report export failures. + The standalone `openshell-driver-docker` binary accepts `OPENSHELL_OTLP_ENDPOINT`. When set, it exports Docker driver spans to that collector, continues W3C trace context from gateway RPC metadata, and flushes diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index b0436634ee..d55d20ec93 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -13,6 +13,10 @@ spans export to the same OTLP/gRPC collector with the service name uses the same compute-driver RPC span names in its in-process and standalone forms. +`mise run gateway:podman` enables this export only when a local collector is +listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration +so the development gateway does not repeatedly report export failures. + Before creating the container, the driver inspects the final sandbox image and captures its immutable image ID and raw OCI `Config.User`. Container creation uses that image ID with pulling disabled, preventing a mutable tag from changing diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 67666be735..f455beaeb5 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -47,6 +47,7 @@ By default `mise run gateway:vm`: - Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`. - Places the VM driver state (per-sandbox `overlay.ext4`, image cache, and `run/compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`. - Writes `.cache/gateway-vm/gateway.toml` with `[openshell.drivers.vm].driver_dir = "$PWD/target/debug"` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`. +- Enables OTLP trace export to `http://127.0.0.1:4317` only when a local collector is listening there. Otherwise, it omits the OTLP configuration to avoid repeated export failures. For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges: diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index a1f9e93e6a..b38bfb7942 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -65,6 +65,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + register_gateway_metadata() { local name=$1 local endpoint=$2 @@ -203,9 +218,6 @@ name = "${GATEWAY_NAME}" compute_drivers = ["docker"] disable_tls = true -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -224,6 +236,8 @@ grpc_endpoint = "${GRPC_ENDPOINT}" supervisor_bin = "${SUPERVISOR_BIN}" EOF +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index b27a48df89..ab166865ef 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -129,6 +129,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + register_gateway_metadata() { local name=$1 local endpoint=$2 @@ -208,9 +223,6 @@ compute_drivers = ["podman"] default_image = "${SANDBOX_IMAGE}" disable_tls = true -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -267,6 +279,8 @@ if [[ -n "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE+x}" ]]; then printf 'proxy_ca_bundle = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE}")" >>"${CONFIG_PATH}" fi +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://${CLI_ENDPOINT_HOST}:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index e847b2b8ec..2e8aecc5d0 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -83,6 +83,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + invoking_user() { if [ -n "${SUDO_USER:-}" ] && [ "${SUDO_USER}" != "root" ]; then printf '%s\n' "${SUDO_USER}" @@ -328,9 +343,6 @@ name = "${GATEWAY_NAME}" compute_drivers = ["vm"] disable_tls = ${DISABLE_TLS} -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -349,6 +361,8 @@ driver_dir = "${DRIVER_DIR}" state_dir = "${VM_DRIVER_STATE_DIR}" EOF +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" save_active_gateway "${GATEWAY_NAME}" diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index 39fed2f4b4..dc8adb9bdf 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -142,12 +142,15 @@ k3d_cluster_exists() { merge_kubeconfig() { require_kubectl - local tmp k3d_cfg merged_dir + local tmp merged_dir tmp="$(mktemp)" k3d kubeconfig get "${CLUSTER_NAME}" >"${tmp}" if [[ -s "${KUBECONFIG_TARGET}" ]]; then - KUBECONFIG="${KUBECONFIG_TARGET}:${tmp}" kubectl config view --flatten >"${tmp}.out" + # Put the freshly generated k3d config first so its cluster, context, and + # user entries replace stale entries with the same names. The API server's + # random host port can change when Docker recreates the load balancer. + KUBECONFIG="${tmp}:${KUBECONFIG_TARGET}" kubectl config view --flatten >"${tmp}.out" mv "${tmp}.out" "${KUBECONFIG_TARGET}" else merged_dir="$(dirname "${KUBECONFIG_TARGET}")" @@ -394,7 +397,8 @@ EOF local lb_port_map="${HOST_LB_PORT}:80@loadbalancer" if k3d_cluster_exists; then - echo "k3d cluster '${CLUSTER_NAME}' already exists; merging kubeconfig." + echo "k3d cluster '${CLUSTER_NAME}' already exists; ensuring it is running." + k3d cluster start "${CLUSTER_NAME}" else echo "Creating k3d cluster '${CLUSTER_NAME}'..." k3d cluster create "${CLUSTER_NAME}" \