diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 6c8bc0f31c..10465d251d 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -141,11 +141,8 @@ jobs: uses: ./.github/workflows/security-scan.yml with: candidate_ref: ${{ inputs.tag || github.ref_name }} - # Temporary observation mode for the pre-existing CodeQL, Trivy, and - # Zizmor backlog. Remove after baseline/delta enforcement is implemented. - # Cargo Deny, Codex Security, and scanner execution errors remain fatal. fail-on-codex-findings: true - fail-on-static-findings: false + fail-on-static-findings: true images: | ghcr.io/nvidia/openshell/gateway:${{ needs.compute-versions.outputs.source_sha }} ghcr.io/nvidia/openshell/sandbox:${{ needs.compute-versions.outputs.source_sha }} @@ -217,39 +214,80 @@ jobs: checkout-ref: ${{ needs.compute-versions.outputs.source_sha }} conformance-artifact-prefix: openshell-conformance - tag-ghcr-release: - name: Tag GHCR Images - needs: [compute-versions, build-images, release] - runs-on: linux-amd64-cpu8 - timeout-minutes: 10 + qualification-result: + name: Release Qualification + if: always() + needs: + - compute-versions + - security + - conformance-integration + - feature-specific-integration + - docker-e2e + - vm-e2e + runs-on: ubuntu-latest + timeout-minutes: 5 permissions: - packages: write + contents: read + outputs: + artifact-name: ${{ steps.summary.outputs.artifact-name }} + current-profile-passed: ${{ steps.summary.outputs.current-profile-passed }} steps: - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ needs.compute-versions.outputs.source_sha || github.sha }} - - name: Tag images with release version + - name: Record qualification result + id: summary env: + CONFORMANCE_RESULT: ${{ needs.conformance-integration.result }} + DOCKER_E2E_RESULT: ${{ needs.docker-e2e.result }} + FEATURE_INTEGRATION_RESULT: ${{ needs.feature-specific-integration.result }} IS_PRERELEASE: ${{ needs.compute-versions.outputs.is_prerelease }} - SOURCE_TAG: ${{ needs.compute-versions.outputs.source_sha }} - VERSION: ${{ needs.compute-versions.outputs.semver }} + SECURITY_RESULT: ${{ needs.security.result }} + SOURCE_SHA: ${{ needs.compute-versions.outputs.source_sha }} + VM_E2E_RESULT: ${{ needs.vm-e2e.result }} run: | set -euo pipefail - REGISTRY="ghcr.io/nvidia/openshell" - for component in gateway sandbox supervisor; do - echo "Tagging ${REGISTRY}/${component}:${SOURCE_TAG} as ${VERSION}..." - docker buildx imagetools create \ - --prefer-index=false \ - -t "${REGISTRY}/${component}:${VERSION}" \ - "${REGISTRY}/${component}:${SOURCE_TAG}" - if [[ "${IS_PRERELEASE}" != "true" ]]; then - echo "Tagging ${REGISTRY}/${component}:${SOURCE_TAG} as latest..." - docker buildx imagetools create \ - --prefer-index=false \ - -t "${REGISTRY}/${component}:latest" \ - "${REGISTRY}/${component}:${SOURCE_TAG}" - fi - done + current_profile_passed=$(tasks/scripts/generate-qualification-summary.sh qualification-summary.json) + artifact_name="qualification-${RELEASE_TAG}-run-${GITHUB_RUN_ID}-attempt-${GITHUB_RUN_ATTEMPT}" + run_url=$(jq -r '.run.url' qualification-summary.json) + + { + echo "artifact-name=${artifact_name}" + echo "current-profile-passed=${current_profile_passed}" + } >> "${GITHUB_OUTPUT}" + + { + echo "## Release qualification" + echo + echo "- Candidate: \`${RELEASE_TAG}\` (\`${SOURCE_SHA}\`)" + echo "- Current qualification profile passed: **${current_profile_passed}**" + echo "- RFC-0014 qualification coverage complete: **false**" + echo "- Run: [${GITHUB_RUN_ID}, attempt ${GITHUB_RUN_ATTEMPT}](${run_url})" + echo + echo "| Suite | Result |" + echo "| --- | --- |" + echo "| Security | ${SECURITY_RESULT} |" + echo "| Conformance integration | ${CONFORMANCE_RESULT} |" + echo "| Feature integration | ${FEATURE_INTEGRATION_RESULT} |" + echo "| Docker E2E | ${DOCKER_E2E_RESULT} |" + echo "| VM E2E | ${VM_E2E_RESULT} |" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload qualification result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ steps.summary.outputs.artifact-name }} + path: qualification-summary.json + retention-days: 90 + if-no-files-found: error + + - name: Require current qualification profile for stable releases + if: needs.compute-versions.outputs.is_prerelease != 'true' && steps.summary.outputs.current-profile-passed != 'true' + run: | + echo "Stable release ${RELEASE_TAG} did not pass the current release-tag-v1 qualification profile." >&2 + exit 1 build-python-wheel: name: Build Python Wheel @@ -321,7 +359,7 @@ jobs: cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} # --------------------------------------------------------------------------- - # Create a tagged GitHub Release with CLI, gateway, driver, and wheels + # Assemble release artifacts. Stable publication remains qualification-gated. # --------------------------------------------------------------------------- release: name: Release @@ -329,11 +367,7 @@ jobs: - compute-versions - package-binaries - build-python-wheel - - conformance-integration - - feature-specific-integration - - docker-e2e - - vm-e2e - - security + - qualification-result - build-deb - build-rpm - build-snap @@ -478,7 +512,8 @@ jobs: --output release/openshell.rb cat release/openshell.rb - - name: Attest VM driver artifacts + - name: Attest release artifacts + id: attest uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1 with: subject-path: | @@ -616,6 +651,104 @@ jobs: retention-days: 90 if-no-files-found: error + # --------------------------------------------------------------------------- + # Publish OCI artifacts. build-images creates the commit-addressed candidate + # images; these jobs add versioned image tags and charts, and publish the + # qualification summary independently from the release artifacts. + # --------------------------------------------------------------------------- + tag-ghcr-release: + name: Tag GHCR Images + needs: [compute-versions, build-images, release] + runs-on: linux-amd64-cpu8 + timeout-minutes: 10 + permissions: + packages: write + steps: + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + + - name: Tag images with release version + env: + IS_PRERELEASE: ${{ needs.compute-versions.outputs.is_prerelease }} + SOURCE_TAG: ${{ needs.compute-versions.outputs.source_sha }} + VERSION: ${{ needs.compute-versions.outputs.semver }} + run: | + set -euo pipefail + REGISTRY="ghcr.io/nvidia/openshell" + for component in gateway sandbox supervisor; do + echo "Tagging ${REGISTRY}/${component}:${SOURCE_TAG} as ${VERSION}..." + docker buildx imagetools create \ + --prefer-index=false \ + -t "${REGISTRY}/${component}:${VERSION}" \ + "${REGISTRY}/${component}:${SOURCE_TAG}" + if [[ "${IS_PRERELEASE}" != "true" ]]; then + echo "Tagging ${REGISTRY}/${component}:${SOURCE_TAG} as latest..." + docker buildx imagetools create \ + --prefer-index=false \ + -t "${REGISTRY}/${component}:latest" \ + "${REGISTRY}/${component}:${SOURCE_TAG}" + fi + done + + release-helm: + name: Release Helm Chart (OCI) + needs: [compute-versions, release, tag-ghcr-release] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ needs.compute-versions.outputs.source_sha }} + + - uses: ./.github/actions/release-helm-oci + with: + chart-version: ${{ needs.compute-versions.outputs.semver }} + app-version: ${{ needs.compute-versions.outputs.semver }} + release-kind: public + + publish-qualification: + name: Publish Qualification Summary (OCI) + if: >- + always() + && needs.release.result == 'success' + && needs.qualification-result.result == 'success' + needs: [compute-versions, qualification-result, release] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: read + contents: read + packages: write + steps: + - name: Download qualification result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ needs.qualification-result.outputs.artifact-name }} + + - name: Install ORAS + uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0 + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | oras login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + + - name: Publish qualification summary + env: + SOURCE_SHA: ${{ needs.compute-versions.outputs.source_sha }} + VERSION: ${{ needs.compute-versions.outputs.semver }} + run: | + set -euo pipefail + ref="ghcr.io/nvidia/openshell/qualification:${VERSION}-run-${GITHUB_RUN_ID}-attempt-${GITHUB_RUN_ATTEMPT}" + oras push "${ref}" \ + --artifact-type application/vnd.nvidia.openshell.qualification.v1 \ + --annotation "org.opencontainers.image.revision=${SOURCE_SHA}" \ + --annotation "org.opencontainers.image.version=${VERSION}" \ + qualification-summary.json:application/vnd.nvidia.openshell.qualification.summary.v1+json + + echo "Published qualification summary to \`${ref}\`." >> "${GITHUB_STEP_SUMMARY}" + publish-fern-docs: name: Sync and Publish Fern Docs needs: [compute-versions, release, publish-sdk-typescript, release-helm, trigger-wheel-publish] @@ -675,25 +808,6 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: mise run sdk:ts:publish - release-helm: - name: Release Helm Chart (OCI) - needs: [compute-versions, release, tag-ghcr-release] - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - packages: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - ref: ${{ needs.compute-versions.outputs.source_sha }} - - - uses: ./.github/actions/release-helm-oci - with: - chart-version: ${{ needs.compute-versions.outputs.semver }} - app-version: ${{ needs.compute-versions.outputs.semver }} - release-kind: public - trigger-wheel-publish: name: Trigger Wheel Publish needs: [compute-versions, release] diff --git a/CI.md b/CI.md index 35802a42dc..1324ca9d7f 100644 --- a/CI.md +++ b/CI.md @@ -97,12 +97,10 @@ Actions or call it from another workflow. All applicable children analyze the candidate snapshot. Cargo Deny uses its existing NVIDIA self-hosted runner and CI container. -Tagged releases temporarily run CodeQL, Trivy, and Zizmor findings in -observation mode while the existing backlog is triaged. Cargo Deny advisories, -Codex Security findings, and scanner execution errors still block publication. -The temporary `fail-on-static-findings: false` override must be removed when -baseline/delta enforcement is implemented; it does not waive Cargo Deny, Codex -Security, or scanner execution failures. +Tagged releases treat CodeQL, Trivy, Zizmor, Cargo Deny, and Codex Security +findings as failures of the currently implemented qualification profile. A +profile failure does not prevent a pre-release candidate's complete artifact +set from being published, but it does prevent stable publication. ```shell gh workflow run security-scan.yml --ref main \ @@ -141,8 +139,20 @@ jobs: ``` Set `needs: security` on a downstream promotion job to require successful scans. -The tagged release workflow does this after publishing its commit-addressed OCI -images, so failed scans or High/Critical findings block release publication. +The tagged release workflow records security and integration outcomes in a +qualification job after publishing its commit-addressed OCI images. A failed +check remains visible in the workflow, but pre-release artifact assembly and +publication continue. Stable publication currently requires the implemented +`release-tag-v1` profile to pass; that profile is an incremental subset of RFC +0014 qualification. + +Each qualification attempt writes its result to the Actions run summary and +uploads `qualification-summary.json` as a 90-day workflow artifact. After the +candidate release is published, the workflow publishes the same summary to +`ghcr.io/nvidia/openshell/qualification:-run--attempt-`. +The summary contains qualification results and policy coverage only; artifact +identity belongs in the release manifest. The run ID and attempt distinguish +reruns without overwriting earlier evidence. `CODEX_SECURITY_API_KEY` is required; `CACHIX_AUTH_TOKEN` is optional. The parent publishes SARIF, including Codex results for manual parent runs. Codex keeps its @@ -388,7 +398,7 @@ These workflows run after merge to publish dev/tagged artifacts and verify them. | File | Role | |---|---| | `.github/workflows/release-dev.yml` | Publishes the rolling `dev` build on every push to `main`. Builds gateway, sandbox, and supervisor images and binaries, packages, wheels, and pushes the Helm chart as `oci://ghcr.io/nvidia/openshell/helm-chart:0.0.0-dev` (plus an immutable `0.0.0-dev.` pin). Also dispatchable manually. | -| `.github/workflows/release-tag.yml` | Publishes tagged stable releases and manually dispatched pre-releases. Its automatic tag trigger excludes `-pre.*`. Security Scan gates release publication on Cargo Deny advisories, Codex Security findings, and scanner execution errors while the static-analysis backlog remains in temporary observation mode. | +| `.github/workflows/release-tag.yml` | Publishes tagged stable releases and manually dispatched pre-releases. Its automatic tag trigger excludes `-pre.*`. Security and integration failures do not block pre-release artifact publication. Stable publication requires the currently implemented qualification profile to pass; the summary identifies the remaining RFC 0014 coverage. | | `.github/workflows/release-canary.yml` | Smoke-tests published dev artifacts on `macos`, `ubuntu`, `fedora`, and `kubernetes` (kind + Helm) runners. Each job reaches its gateway and creates, exercises, and deletes a sandbox. It runs automatically after `Release Dev` succeeds and supports manual dispatch (`gh workflow run release-canary.yml --ref `). See the `test-release-canary` skill for the playbook and local kind reproduction. | ## Required status contexts diff --git a/architecture/build.md b/architecture/build.md index 0ebb9d2770..081f11af7b 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -503,21 +503,32 @@ job republishes the analysis job's outcome as the `OpenShell / Codex Security (informational)` status. None of these checks are required statuses, so they do not gate merges. -Codex Security findings are informational during the observation phase, and the -workflow only reports on candidates that already exist. Gating stable promotion -on qualification results remains proposed in -[RFC 0014](../rfc/0014-release-stability/release-qualification.md). +The workflow only reports on candidates that already exist. It incrementally +implements the qualification model from +[RFC 0014](../rfc/0014-release-stability/release-qualification.md): failed +checks do not prevent publication of an immutable pre-release candidate. The +summary explicitly records that the current profile does not yet provide the +RFC's complete qualification coverage. The tagged release workflow calls the aggregate Security Scan after publishing the candidate's commit-addressed gateway, sandbox, and supervisor images. CodeQL, Trivy, Cargo Deny, and Actionlint/Zizmor run for every release tag; Codex Security -also runs for pre-release tags. The release job depends on the aggregate result, -which fails on scanner errors, Cargo Deny advisories, and Codex Security -findings. CodeQL, Trivy, and Zizmor findings temporarily remain informational -while their existing backlog is triaged. Remove the release workflow's -`fail-on-static-findings: false` override when baseline/delta enforcement is -implemented so new High or Critical findings become blocking without requiring -the historical backlog to reach zero first. +also runs for pre-release tags. High or Critical findings and scanner failures +fail qualification. + +The `Release Qualification` job aggregates security, conformance, feature, +Docker E2E, and VM E2E results. The currently implemented profile gates stable +publication, but it does not represent complete RFC 0014 qualification. For a +pre-release it records a failed result without blocking artifact assembly, +image tagging, or Helm publication; the failing underlying suite keeps the +workflow visibly red. Every attempt writes a summary to the Actions run summary +and a 90-day Actions artifact. After release assembly succeeds, the workflow +publishes the same result to +`ghcr.io/nvidia/openshell/qualification:-run--attempt-`. +`tasks/scripts/generate-qualification-summary.sh` generates qualification +metadata only. Artifact identity remains the responsibility of the separate +release manifest. Including both the run ID and attempt preserves the result of +each rerun. `release-auto-tag.yml` runs at 14:00 Europe/Zurich on weekdays (including daylight saving time changes) and supports manual dispatch. Maintainers start weekday diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index a5edb2accc..ba82e39726 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -673,8 +673,8 @@ pub enum Request { sandbox_id: String, spec: AgentSpecWire, policy: Box, - ca_cert: Option>, - ca_bundle: Option>, + ca_cert: Option, + ca_bundle: Option, provider_env_revision: u64, provider_env: std::collections::HashMap, }, @@ -1536,8 +1536,8 @@ mod tests { landlock: LandlockPolicy::default(), process: ProcessPolicy::default(), })), - ca_cert: Some(b"test certificate".to_vec()), - ca_bundle: Some(b"test bundle".to_vec()), + ca_cert: Some("test certificate".to_string()), + ca_bundle: Some("test bundle".to_string()), provider_env_revision: 7, provider_env: std::collections::HashMap::from([( "OPENAI_API_KEY".to_string(), @@ -1590,6 +1590,35 @@ mod tests { )); } + #[test] + fn start_agent_with_large_ca_bundle_fits_in_frame_limit() { + let request = RequestEnvelope::new(Request::StartAgent { + sandbox_id: "sandbox-1".to_string(), + spec: AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: None, + timeout_secs: 5, + interactive: false, + }, + policy: Box::new(SandboxPolicyWire::from(SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + })), + ca_cert: Some("A".repeat(16 * 1024)), + ca_bundle: Some("B".repeat(400 * 1024)), + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + }) + .expect("request envelope"); + let frame = encode_frame(&request).expect("large CA bundle must fit in frame limit"); + let decoded: RequestEnvelope = decode_frame(&frame).expect("round-trip"); + assert_eq!(decoded, request); + } + #[test] fn rejects_declared_oversize() { let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32"); diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 39bdd77aba..a6854d52c2 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -371,15 +371,17 @@ impl ReadyBoundary for RemoteReady { .unwrap_or_else(std::sync::PoisonError::into_inner) .clone(); let (ca_cert, ca_bundle) = if let Some((ca_cert, ca_bundle)) = ca_paths { - let ca_cert = tokio::fs::read(&ca_cert).await.map_err(|error| { + let ca_cert = tokio::fs::read_to_string(&ca_cert).await.map_err(|error| { BackendError::Process(format!("read host proxy CA {}: {error}", ca_cert.display())) })?; - let ca_bundle = tokio::fs::read(&ca_bundle).await.map_err(|error| { - BackendError::Process(format!( - "read host proxy CA bundle {}: {error}", - ca_bundle.display() - )) - })?; + let ca_bundle = tokio::fs::read_to_string(&ca_bundle) + .await + .map_err(|error| { + BackendError::Process(format!( + "read host proxy CA bundle {}: {error}", + ca_bundle.display() + )) + })?; (Some(ca_cert), Some(ca_bundle)) } else { (None, None) @@ -3169,8 +3171,8 @@ mod tests { sandbox_id: context.sandbox_id, spec: AgentSpecWire::from(context.agent), policy: Box::new(SandboxPolicyWire::from(context.policy)), - ca_cert: Some(vec![b'c'; 16 * 1024]), - ca_bundle: Some(vec![b'b'; 256 * 1024]), + ca_cert: Some("c".repeat(16 * 1024)), + ca_bundle: Some("b".repeat(256 * 1024)), provider_env_revision: 0, provider_env: HashMap::new(), }) @@ -3228,8 +3230,8 @@ mod tests { sandbox_id: context.sandbox_id, spec: AgentSpecWire::from(context.agent), policy: Box::new(SandboxPolicyWire::from(context.policy)), - ca_cert: Some(vec![b'c'; 16 * 1024]), - ca_bundle: Some(vec![b'b'; 256 * 1024]), + ca_cert: Some("c".repeat(16 * 1024)), + ca_bundle: Some("b".repeat(256 * 1024)), provider_env_revision: 0, provider_env: HashMap::new(), }) diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 9b10d1b728..9d1f477241 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -1403,8 +1403,8 @@ mod linux { sandbox_id: String, spec: AgentSpecWire, policy: SandboxPolicyWire, - ca_cert: Option>, - ca_bundle: Option>, + ca_cert: Option, + ca_bundle: Option, provider_env_revision: u64, provider_env: std::collections::HashMap, } @@ -2398,8 +2398,8 @@ mod linux { sandbox_id: String, spec: AgentSpecWire, policy: SandboxPolicyWire, - ca_cert: Option>, - ca_bundle: Option>, + ca_cert: Option, + ca_bundle: Option, provider_env_revision: u64, provider_env: std::collections::HashMap, ) -> Response { @@ -2678,8 +2678,8 @@ mod linux { } fn install_ca_material( - ca_cert: Option>, - ca_bundle: Option>, + ca_cert: Option, + ca_bundle: Option, ) -> Result, String> { let (ca_cert, ca_bundle) = match (ca_cert, ca_bundle) { (Some(ca_cert), Some(ca_bundle)) => (ca_cert, ca_bundle), @@ -2692,8 +2692,8 @@ mod linux { }; install_ca_material_at( Path::new(openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR), - &ca_cert, - &ca_bundle, + ca_cert.as_bytes(), + ca_bundle.as_bytes(), ) } diff --git a/tasks/scripts/generate-qualification-summary.sh b/tasks/scripts/generate-qualification-summary.sh new file mode 100755 index 0000000000..1c5be1f20c --- /dev/null +++ b/tasks/scripts/generate-qualification-summary.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +die() { + echo "generate-qualification-summary: $*" >&2 + exit 1 +} + +require_env() { + local name + for name in "$@"; do + if [[ -z "${!name:-}" ]]; then + die "required environment variable ${name} is not set" + fi + done +} + +[[ $# -eq 1 ]] || die "usage: $0 OUTPUT" +output=$1 +current_profile_passed=true + +require_env \ + RELEASE_TAG SOURCE_SHA IS_PRERELEASE GITHUB_RUN_ID GITHUB_RUN_ATTEMPT \ + GITHUB_SERVER_URL GITHUB_REPOSITORY SECURITY_RESULT CONFORMANCE_RESULT \ + FEATURE_INTEGRATION_RESULT DOCKER_E2E_RESULT VM_E2E_RESULT + +for result in \ + "${SECURITY_RESULT}" \ + "${CONFORMANCE_RESULT}" \ + "${FEATURE_INTEGRATION_RESULT}" \ + "${DOCKER_E2E_RESULT}" \ + "${VM_E2E_RESULT}"; do + if [[ "${result}" != "success" ]]; then + current_profile_passed=false + break + fi +done + +mkdir -p "$(dirname "${output}")" +jq -n \ + --arg tag "${RELEASE_TAG}" \ + --arg source_sha "${SOURCE_SHA}" \ + --arg is_prerelease "${IS_PRERELEASE}" \ + --argjson current_profile_passed "${current_profile_passed}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg run_attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg run_url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" \ + --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg security "${SECURITY_RESULT}" \ + --arg conformance "${CONFORMANCE_RESULT}" \ + --arg feature_integration "${FEATURE_INTEGRATION_RESULT}" \ + --arg docker_e2e "${DOCKER_E2E_RESULT}" \ + --arg vm_e2e "${VM_E2E_RESULT}" \ + '{ + schema_version: 1, + tag: $tag, + source_sha: $source_sha, + is_prerelease: ($is_prerelease == "true"), + profile: { + name: "release-tag-v1", + rfc_0014_complete: false, + missing_suites: [ + "upgrade", + "breaking API change review", + "remaining RFC conformance configurations" + ] + }, + current_profile_passed: $current_profile_passed, + run: { + id: $run_id, + attempt: $run_attempt, + url: $run_url + }, + generated_at: $generated_at, + suites: { + security: $security, + conformance_integration: $conformance, + feature_integration: $feature_integration, + docker_e2e: $docker_e2e, + vm_e2e: $vm_e2e + } + }' > "${output}" + +printf '%s\n' "${current_profile_passed}" diff --git a/tasks/scripts/test-qualification-summary.sh b/tasks/scripts/test-qualification-summary.sh new file mode 100755 index 0000000000..a2272d11ee --- /dev/null +++ b/tasks/scripts/test-qualification-summary.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +GENERATOR="${SCRIPT_DIR}/generate-qualification-summary.sh" +TEST_DIR=$(mktemp -d) +trap 'rm -rf "${TEST_DIR}"' EXIT + +export RELEASE_TAG=v0.1.0-pre.8 +export SOURCE_SHA=0123456789abcdef0123456789abcdef01234567 +export IS_PRERELEASE=true +export GITHUB_REPOSITORY=NVIDIA/OpenShell +export GITHUB_SERVER_URL=https://github.com +export GITHUB_RUN_ID=1234 +export GITHUB_RUN_ATTEMPT=2 +export SECURITY_RESULT=failure +export CONFORMANCE_RESULT=success +export FEATURE_INTEGRATION_RESULT=success +export DOCKER_E2E_RESULT=success +export VM_E2E_RESULT=success + +current_profile_passed=$("${GENERATOR}" "${TEST_DIR}/qualification-summary.json") +[[ "${current_profile_passed}" == "false" ]] +jq -e ' + .schema_version == 1 + and .tag == "v0.1.0-pre.8" + and .source_sha == "0123456789abcdef0123456789abcdef01234567" + and .is_prerelease == true + and .current_profile_passed == false + and .profile.rfc_0014_complete == false + and .suites.security == "failure" + and .run.id == "1234" + and .run.attempt == "2" +' "${TEST_DIR}/qualification-summary.json" >/dev/null + +export SECURITY_RESULT=success +current_profile_passed=$("${GENERATOR}" "${TEST_DIR}/qualified-summary.json") +[[ "${current_profile_passed}" == "true" ]] +jq -e '.current_profile_passed == true' "${TEST_DIR}/qualified-summary.json" >/dev/null + +echo "qualification summary tests passed" diff --git a/tasks/test.toml b/tasks/test.toml index 74166db440..701bab7327 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -16,6 +16,7 @@ depends = [ "test:gateway-config", "test:e2e-parity", "test:packaging-assets", + "test:qualification-summary", "test:codex-security-release-range", "test:docs-website", ] @@ -55,6 +56,12 @@ run = "tasks/scripts/test-packaging-assets.sh" run_windows = "echo Skipping test:packaging-assets: Linux service and RPM assets do not apply on Windows." hide = true +["test:qualification-summary"] +description = "Test release qualification summary generation" +run = "tasks/scripts/test-qualification-summary.sh" +run_windows = "echo Skipping test:qualification-summary: the release workflow runs on Unix hosts." +hide = true + ["test:codex-security-release-range"] description = "Test Codex Security release-range resolution" run = "uv run --no-project --with pytest pytest -o \"python_files=*_test.py\" tasks/scripts/codex_security_range_test.py"