Skip to content

Commit 7ee8a9c

Browse files
ci(tests): run test plans from a reusable E2E Suites workflow
Add a reusable workflow that takes a plan. A planning job runs `openshell-test-runner test --plan <plan> --emit-github-matrix`, and one job per environment, on the runner label the matrix gives, runs `nix run .#test-runner -- test --plan <plan> --env <id> --strict`. The jobs use the same integration inputs as the existing tmachine lanes, cache the tmachine disks keyed on the Ansible tree, upload the JUnit reports and logs whether or not the run passes, and an aggregator job reports the overall result. Branch E2E calls it for tests/plans/cpu-tests.yaml when the test:e2e label is set. The job is not in the Core E2E result's needs yet, so it starts out advisory. The existing conformance, feature and driver lanes are unchanged. Modeled on the E2E Suites workflow in #4629. Co-authored-by: Evan Lezar <elezar@nvidia.com> Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
1 parent f8088fd commit 7ee8a9c

2 files changed

Lines changed: 169 additions & 0 deletions

File tree

‎.github/workflows/branch-e2e.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,22 @@ jobs:
303303
{"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"driver-podman"}
304304
]
305305
306+
# Run the tests in tests/plans/cpu-tests.yaml. The plan decides which suites
307+
# run in which environments, so adding coverage means editing the plan, not
308+
# this workflow. Not yet in core-e2e-result.needs: it starts out advisory.
309+
plan-cpu-tests:
310+
needs: [pr_metadata, prepare-integration]
311+
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
312+
permissions:
313+
actions: read
314+
contents: read
315+
packages: read
316+
uses: ./.github/workflows/e2e-suites.yml
317+
with:
318+
plan: cpu-tests
319+
source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
320+
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
321+
306322
docker-e2e:
307323
needs: [pr_metadata, build-binaries, build-images]
308324
if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'

‎.github/workflows/e2e-suites.yml‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
# Runs a test plan (tests/plans/<plan>.yaml) on CI. The plan decides which
5+
# suites run in which environments; this workflow keeps no list of either.
6+
# `openshell-test-runner test --emit-github-matrix` turns the plan into one job per
7+
# environment, and each job runs the plan's tests in that environment with
8+
# `openshell-test-runner test --env <id> --strict`.
9+
#
10+
# Modeled on the E2E Suites workflow in NVIDIA/OpenShell#4629 by Evan Lezar.
11+
12+
name: E2E Suites
13+
14+
on:
15+
workflow_call:
16+
inputs:
17+
plan:
18+
description: Plan to run, the file stem of a plan in tests/plans/
19+
required: true
20+
type: string
21+
source-sha:
22+
description: Source revision of the prepared candidate artifacts
23+
required: true
24+
type: string
25+
integration-inputs-artifact-id:
26+
description: Immutable ID of the prepared integration inputs artifact
27+
required: true
28+
type: string
29+
30+
permissions:
31+
actions: read
32+
contents: read
33+
packages: read
34+
35+
# github.workflow is the caller's name here, so scope the group to this plan to
36+
# avoid colliding with (and cancelling) the calling workflow's other jobs.
37+
concurrency:
38+
group: ${{ github.workflow }}-e2e-suites-${{ inputs.plan }}-${{ github.ref }}
39+
cancel-in-progress: true
40+
41+
jobs:
42+
plan:
43+
name: Plan ${{ inputs.plan }}
44+
runs-on: ubuntu-24.04
45+
timeout-minutes: 30
46+
outputs:
47+
matrix: ${{ steps.matrix.outputs.matrix }}
48+
steps:
49+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
50+
with:
51+
ref: ${{ inputs['source-sha'] }}
52+
persist-credentials: false
53+
54+
- uses: ./.github/actions/setup-nix
55+
with:
56+
prepare-shell: "true"
57+
58+
- name: Generate the job matrix from the plan
59+
id: matrix
60+
shell: nix develop -c bash -euo pipefail {0}
61+
env:
62+
PLAN: ${{ inputs.plan }}
63+
run: |
64+
matrix=$(cargo run --locked --quiet \
65+
--manifest-path tests/runner/Cargo.toml --bin openshell-test-runner -- \
66+
test --plan "tests/plans/${PLAN}.yaml" --emit-github-matrix)
67+
echo "$matrix"
68+
echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
69+
70+
suites:
71+
name: E2E suites (${{ matrix.plan }}, ${{ matrix.environment }})
72+
needs: plan
73+
strategy:
74+
fail-fast: false
75+
matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
76+
runs-on: ${{ matrix.runner }}
77+
timeout-minutes: 90
78+
steps:
79+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
80+
with:
81+
ref: ${{ inputs['source-sha'] }}
82+
persist-credentials: false
83+
84+
- name: Enable KVM access
85+
run: |
86+
set -euo pipefail
87+
if [[ ! -c /dev/kvm ]]; then
88+
echo "::error::The runner did not expose /dev/kvm"
89+
exit 1
90+
fi
91+
sudo chmod 0666 /dev/kvm
92+
exec 3<>/dev/kvm
93+
exec 3>&-
94+
95+
- uses: ./.github/actions/setup-nix
96+
97+
- name: Cache tmachine disks
98+
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
99+
with:
100+
path: ~/.cache/tmachine
101+
# tmachine keys its disk layers on a hash of the Ansible tree.
102+
# actions/cache doesn't save after an exact key hit, so a fixed key
103+
# would rebuild the changed layers on every run. Key on the playbooks
104+
# and fall back to the newest entry for the same runner.
105+
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }}
106+
restore-keys: |
107+
tmachine-${{ runner.os }}-${{ runner.arch }}-
108+
109+
- name: Download integration inputs
110+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
111+
with:
112+
artifact-ids: ${{ inputs['integration-inputs-artifact-id'] }}
113+
path: artifacts
114+
115+
- name: Run the plan in this environment
116+
env:
117+
PLAN: ${{ matrix.plan }}
118+
ENVIRONMENT: ${{ matrix.environment }}
119+
run: |
120+
# Retry dependency preparation, then run the plan once. The runner
121+
# builds the suites for the guest, so it needs no prepared archives.
122+
# --strict fails the job instead of skipping an environment or a
123+
# selection that would run no tests.
124+
nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner
125+
nix run .#openshell-test-runner -- test \
126+
--plan "tests/plans/${PLAN}.yaml" \
127+
--env "${ENVIRONMENT}" \
128+
--strict \
129+
--results artifacts/test-results
130+
131+
# The results directory holds each family's JUnit report (junit.xml),
132+
# the nextest log, and the guest's diagnostics when a run failed.
133+
- name: Upload results
134+
if: always()
135+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
136+
with:
137+
name: e2e-results-${{ matrix.plan }}-${{ matrix.environment }}
138+
path: artifacts/test-results
139+
if-no-files-found: ignore
140+
retention-days: 7
141+
142+
result:
143+
name: E2E suites result (${{ inputs.plan }})
144+
needs: [plan, suites]
145+
if: always()
146+
runs-on: ubuntu-24.04
147+
steps:
148+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
149+
with:
150+
persist-credentials: false
151+
- uses: ./.github/actions/check-job-results
152+
with:
153+
results: ${{ toJSON(needs) }}

0 commit comments

Comments
 (0)