Skip to content

Commit b1ca5f3

Browse files
feat(test-runner): start environment services after boot
An environment can list services, such as keycloak. After the VM boots, the runner applies each service's playbook, reads the OPENSHELL_TEST_* variables it publishes in /etc/openshell-test/env.d/<service>.env, and passes them to testkit-ready and guest nextest. Adds a Docker environment with Keycloak. Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
1 parent ae0268d commit b1ca5f3

7 files changed

Lines changed: 307 additions & 10 deletions

File tree

‎tests/environments/docker.yaml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33

44
# Docker-backed environments, by Evan Lezar (NVIDIA/OpenShell#4629).
55
#
6-
# The ID must equal <backend>.<os><version without dots>-<arch>.<runtime>.<installer>.
6+
# The ID must equal <backend>.<os><version without dots>-<arch>.<runtime>.<installer>,
7+
# followed by +<service> for each entry in `services`.
78
# `tmachine` names the environment and installer in tests/tmachine/config.nix
89
# that provision it. `gateway` is the name the installer registers the gateway
910
# under with the CLI (`openshell gateway add --name`); the packaged installers
@@ -22,3 +23,15 @@ environments:
2223
installer: deb
2324
tmachine: { environment: ubuntu-docker-rootful, installer: deb }
2425
gateway: openshell
26+
27+
# The same VM with Keycloak started after boot, for `features` suites that
28+
# need an OIDC issuer. `services` adds `+<service>` to the ID; see
29+
# tests/runner/src/services.rs.
30+
- id: tmachine.ubuntu2404-amd64.docker-rootful.deb+keycloak
31+
backend: tmachine
32+
machine: { os: ubuntu, version: "24.04", arch: amd64 }
33+
runtime: docker-rootful
34+
installer: deb
35+
tmachine: { environment: ubuntu-docker-rootful, installer: deb }
36+
gateway: openshell
37+
services: [keycloak]

‎tests/plans/README.md‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,23 @@ replaced by `-`: `tests/file_transfer.rs` is the suite `file-transfer`.
1919

2020
| Plan | Runs |
2121
|---|---|
22-
| `cpu-tests.yaml` | The `conformance` family on Ubuntu 24.04 amd64 with rootful Docker and the Debian package |
22+
| `cpu-tests.yaml` | The `conformance` family on Ubuntu 24.04 amd64 with rootful Docker and the Debian package, and the `features` suites that need Keycloak on the same VM with Keycloak started |
23+
24+
## Environment services
25+
26+
A suite that needs something besides the gateway, such as an OIDC issuer, runs
27+
in an environment that lists it under `services`. The environment ID ends in
28+
`+<service>` for each one, e.g. `tmachine.ubuntu2404-amd64.docker-rootful.deb+keycloak`.
29+
30+
After the VM boots, the runner applies
31+
`tests/tmachine/ansible/playbooks/services/<service>.yaml`. The playbook starts
32+
the service and writes `OPENSHELL_TEST_*` lines to
33+
`/etc/openshell-test/env.d/<service>.env`; the runner passes them to the tests.
34+
A suite-local fixture reads them, as `tests/suites/features/keycloak` does. To
35+
run such a suite against your own deployment, set the same variables yourself.
36+
37+
To add a service, add it to `Service` in `tests/runner/src/services.rs`, write
38+
its playbook, and add an environment that lists it.
2339

2440
## How a plan runs on CI
2541

‎tests/runner/src/environments.rs‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ use std::path::Path;
1717
use anyhow::{Context, Result, bail, ensure};
1818
use serde::Deserialize;
1919

20+
use crate::services::Service;
21+
2022
/// All environments, keyed by ID.
2123
#[derive(Debug, Default)]
2224
pub struct Catalog {
@@ -45,6 +47,10 @@ pub struct Environment {
4547
/// use the CLI's active gateway.
4648
#[serde(default)]
4749
pub gateway: Option<String>,
50+
/// Services the runner starts after boot, such as Keycloak. See
51+
/// [`crate::services`].
52+
#[serde(default)]
53+
pub services: Vec<Service>,
4854
}
4955

5056
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
@@ -109,15 +115,20 @@ impl Arch {
109115
impl Environment {
110116
/// The ID implied by the environment's fields.
111117
pub fn expected_id(&self) -> String {
112-
format!(
118+
let mut id = format!(
113119
"{}.{}{}-{}.{}.{}",
114120
self.backend,
115121
self.machine.os,
116122
self.machine.version.replace('.', ""),
117123
self.machine.arch,
118124
self.runtime,
119125
self.installer
120-
)
126+
);
127+
for service in &self.services {
128+
id.push('+');
129+
id.push_str(&service.to_string());
130+
}
131+
id
121132
}
122133

123134
fn validate(&self) -> Result<()> {
@@ -242,6 +253,25 @@ environments:
242253
assert_eq!(env.gateway.as_deref(), Some("openshell"));
243254
}
244255

256+
#[test]
257+
fn services_are_part_of_the_id() {
258+
let yaml = UBUNTU_DOCKER
259+
.replace("docker-rootful.deb\n", "docker-rootful.deb+keycloak\n")
260+
.replace(" runtime:", " services: [keycloak]\n runtime:");
261+
let environments = catalog(&yaml).unwrap();
262+
let env = environments
263+
.get("tmachine.ubuntu2404-amd64.docker-rootful.deb+keycloak")
264+
.unwrap();
265+
assert_eq!(env.services, [Service::Keycloak]);
266+
267+
let without_suffix = yaml.replace("deb+keycloak\n", "deb\n");
268+
let err = catalog(&without_suffix).unwrap_err();
269+
assert!(
270+
err.to_string().contains("does not match its fields"),
271+
"{err}"
272+
);
273+
}
274+
245275
#[test]
246276
fn rejects_id_that_does_not_match_fields() {
247277
let yaml = UBUNTU_DOCKER.replace("docker-rootful.deb", "docker-rootless.deb");

‎tests/runner/src/execute.rs‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ use tmachine::config::Config;
2222
use crate::bundle::{Bundle, PROFILE, READY_BINARY};
2323
use crate::jobs::{FamilyJob, Job};
2424
use crate::report::{Outcome, Status, annotate_junit, verify_junit};
25+
use crate::services;
2526

2627
const GUEST_ROOT: &str = "/var/tmp/openshell-tests";
2728
/// How long the gateway has to become healthy after the environment is up.
@@ -80,6 +81,32 @@ pub async fn run(job: &Job, options: &Options<'_>) -> Vec<Outcome> {
8081
}
8182
};
8283

84+
let service_env = if job.environment.services.is_empty() {
85+
Ok(Vec::new())
86+
} else {
87+
eprintln!("==> {env_id}: starting services");
88+
services::start(&session, &job.environment.services).await
89+
};
90+
let service_env = match service_env {
91+
Ok(vars) => vars,
92+
Err(error) => {
93+
let message = format!("start services: {error:#}");
94+
eprintln!("==> {env_id}: {message}");
95+
if let Err(error) = session
96+
.collect_diagnostics(&options.results.join(env_id).join("diagnostics"))
97+
.await
98+
{
99+
eprintln!("==> {env_id}: collecting diagnostics failed: {error:#}");
100+
}
101+
session.down().await;
102+
return job
103+
.families
104+
.iter()
105+
.map(|family| outcome(family, Status::Error(message.clone())))
106+
.collect();
107+
}
108+
};
109+
83110
let mut outcomes = Vec::new();
84111
for family in &job.families {
85112
if !family.has_tests() {
@@ -92,7 +119,7 @@ pub async fn run(job: &Job, options: &Options<'_>) -> Vec<Outcome> {
92119
family.family,
93120
family.filterset()
94121
);
95-
let status = match run_family(&session, job, family, options, &dir).await {
122+
let status = match run_family(&session, job, family, &service_env, options, &dir).await {
96123
Ok(status) => status,
97124
Err(error) => Status::Error(format!("{error:#}")),
98125
};
@@ -149,13 +176,14 @@ async fn provision(job: &Job, config: &Config) -> Result<Session> {
149176

150177
/// The environment variables that point the tests at the system under test:
151178
/// the registered gateway (when the environment names one), the installed
152-
/// CLI.
153-
pub fn guest_env(job: &Job, openshell_bin: &str) -> Vec<String> {
179+
/// CLI, and whatever the environment's services published.
180+
pub fn guest_env(job: &Job, openshell_bin: &str, service_env: &[String]) -> Vec<String> {
154181
let mut vars = Vec::new();
155182
if let Some(gateway) = &job.environment.gateway {
156183
vars.push(format!("OPENSHELL_GATEWAY={gateway}"));
157184
}
158185
vars.push(format!("OPENSHELL_BIN={openshell_bin}"));
186+
vars.extend(service_env.iter().cloned());
159187
vars
160188
}
161189

@@ -208,6 +236,7 @@ async fn run_family(
208236
session: &Session,
209237
job: &Job,
210238
family: &FamilyJob,
239+
service_env: &[String],
211240
options: &Options<'_>,
212241
dir: &Path,
213242
) -> Result<Status> {
@@ -229,7 +258,7 @@ async fn run_family(
229258
!openshell_bin.is_empty(),
230259
"the installer did not put openshell on the guest's PATH"
231260
);
232-
let env = guest_env(job, openshell_bin);
261+
let env = guest_env(job, openshell_bin, service_env);
233262

234263
let guest = format!("{GUEST_ROOT}/{}", family.family);
235264
checked(
@@ -448,17 +477,29 @@ mod tests {
448477
families: Vec::new(),
449478
};
450479
assert_eq!(
451-
guest_env(&job, "/usr/bin/openshell"),
480+
guest_env(&job, "/usr/bin/openshell", &[]),
481+
[
482+
"OPENSHELL_GATEWAY=openshell",
483+
"OPENSHELL_BIN=/usr/bin/openshell",
484+
]
485+
);
486+
assert_eq!(
487+
guest_env(
488+
&job,
489+
"/usr/bin/openshell",
490+
&["OPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180".to_string()]
491+
),
452492
[
453493
"OPENSHELL_GATEWAY=openshell",
454494
"OPENSHELL_BIN=/usr/bin/openshell",
495+
"OPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180",
455496
]
456497
);
457498

458499
let mut job = job;
459500
job.environment.gateway = None;
460501
assert!(
461-
!guest_env(&job, "/usr/bin/openshell")
502+
!guest_env(&job, "/usr/bin/openshell", &[])
462503
.iter()
463504
.any(|var| var.starts_with("OPENSHELL_GATEWAY="))
464505
);

‎tests/runner/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@ pub mod github;
1313
pub mod jobs;
1414
pub mod plan;
1515
pub mod report;
16+
pub mod services;

‎tests/runner/src/services.rs‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Services an environment runs alongside `OpenShell`, such as Keycloak.
5+
//!
6+
//! The runner starts each service after the VM boots by applying its tmachine
7+
//! playbook. The playbook starts the service, waits until it's ready, and
8+
//! publishes what tests need to reach it as `OPENSHELL_TEST_*` variables in
9+
//! `/etc/openshell-test/env.d/<service>.env`. The runner passes those
10+
//! variables to the tests; suite fixtures read them. Tests don't depend on
11+
//! tmachine: anyone running a suite against their own deployment sets the
12+
//! same variables.
13+
14+
use std::collections::BTreeMap;
15+
use std::fmt;
16+
use std::path::PathBuf;
17+
18+
use anyhow::{Context, Result, bail, ensure};
19+
use serde::Deserialize;
20+
use tmachine::Session;
21+
22+
/// Where service playbooks publish the variables tests read.
23+
pub const ENV_DIR: &str = "/etc/openshell-test/env.d";
24+
/// The prefix every published variable must have, so a service can't
25+
/// override how tests find the gateway or the CLI.
26+
pub const ENV_PREFIX: &str = "OPENSHELL_TEST_";
27+
28+
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
29+
#[serde(rename_all = "lowercase")]
30+
pub enum Service {
31+
Keycloak,
32+
}
33+
34+
impl fmt::Display for Service {
35+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
36+
f.write_str(match self {
37+
Self::Keycloak => "keycloak",
38+
})
39+
}
40+
}
41+
42+
impl Service {
43+
/// The playbook that starts the service, relative to `tests/tmachine`,
44+
/// the runner's working directory.
45+
fn playbook(self) -> PathBuf {
46+
PathBuf::from(format!("ansible/playbooks/services/{self}.yaml"))
47+
}
48+
49+
/// Files the playbook needs from the checkout, relative to
50+
/// `tests/tmachine`.
51+
fn inputs(self) -> BTreeMap<String, PathBuf> {
52+
match self {
53+
Self::Keycloak => BTreeMap::from([(
54+
"keycloak_realm_file".to_string(),
55+
PathBuf::from("../../scripts/keycloak-realm.json"),
56+
)]),
57+
}
58+
}
59+
60+
fn env_file(self) -> String {
61+
format!("{ENV_DIR}/{self}.env")
62+
}
63+
}
64+
65+
/// Start `services` in the booted VM and return the variables they publish,
66+
/// as `KEY=VALUE` strings.
67+
pub async fn start(session: &Session, services: &[Service]) -> Result<Vec<String>> {
68+
let mut vars = Vec::new();
69+
for &service in services {
70+
session
71+
.run_playbooks(&[service.playbook()], &service.inputs())
72+
.await
73+
.with_context(|| format!("start {service}"))?;
74+
let path = service.env_file();
75+
let output = session.exec(&["cat", &path]).await?;
76+
ensure!(
77+
output.status.success(),
78+
"{service} did not publish {path}: {}",
79+
String::from_utf8_lossy(&output.stderr).trim()
80+
);
81+
let contents =
82+
String::from_utf8(output.stdout).with_context(|| format!("{path} is not UTF-8"))?;
83+
vars.extend(parse_env(&path, &contents)?);
84+
}
85+
Ok(vars)
86+
}
87+
88+
/// Parse a published env file: `KEY=VALUE` lines, blank lines and `#`
89+
/// comments. Values are taken literally; there is no quoting. Errors name
90+
/// the key, never the value, since values can be credentials.
91+
pub fn parse_env(path: &str, contents: &str) -> Result<Vec<String>> {
92+
let mut vars = Vec::new();
93+
for (number, line) in contents.lines().enumerate() {
94+
let line = line.trim_end();
95+
if line.trim().is_empty() || line.trim_start().starts_with('#') {
96+
continue;
97+
}
98+
let Some((key, _value)) = line.split_once('=') else {
99+
bail!("{path}:{}: expected KEY=VALUE", number + 1);
100+
};
101+
ensure!(
102+
key.starts_with(ENV_PREFIX)
103+
&& key.len() > ENV_PREFIX.len()
104+
&& key
105+
.bytes()
106+
.all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),
107+
"{path}:{}: {key:?} is not an {ENV_PREFIX}* variable name",
108+
number + 1
109+
);
110+
vars.push(line.to_string());
111+
}
112+
Ok(vars)
113+
}
114+
115+
#[cfg(test)]
116+
mod tests {
117+
use super::*;
118+
119+
#[test]
120+
fn parses_published_variables() {
121+
let vars = parse_env(
122+
"keycloak.env",
123+
"# Published by keycloak.yaml\n\nOPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180/realms/openshell\nOPENSHELL_TEST_KEYCLOAK_PASSWORD=a=b\n",
124+
)
125+
.unwrap();
126+
assert_eq!(
127+
vars,
128+
[
129+
"OPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180/realms/openshell",
130+
"OPENSHELL_TEST_KEYCLOAK_PASSWORD=a=b",
131+
]
132+
);
133+
}
134+
135+
#[test]
136+
fn refuses_variables_outside_the_prefix() {
137+
let err = parse_env("x.env", "OPENSHELL_GATEWAY=other\n").unwrap_err();
138+
assert!(err.to_string().contains("OPENSHELL_GATEWAY"), "{err}");
139+
}
140+
141+
#[test]
142+
fn errors_do_not_echo_values() {
143+
let err = parse_env("x.env", "openshell_test_secret=hunter2\n").unwrap_err();
144+
assert!(!err.to_string().contains("hunter2"), "{err}");
145+
let err = parse_env("x.env", "hunter2\n").unwrap_err();
146+
assert!(!err.to_string().contains("hunter2"), "{err}");
147+
}
148+
149+
#[test]
150+
fn services_are_read_from_yaml() {
151+
let services: Vec<Service> = serde_yml::from_str("[keycloak]").unwrap();
152+
assert_eq!(services, [Service::Keycloak]);
153+
assert_eq!(
154+
Service::Keycloak.playbook(),
155+
PathBuf::from("ansible/playbooks/services/keycloak.yaml")
156+
);
157+
}
158+
}

0 commit comments

Comments
 (0)