refactor(test-runner): provide Keycloak from a tmachine environment

Declare Keycloak as part of a tmachine environment instead of a service
the runner starts after boot. ubuntu-docker-rootful-keycloak's setup runs
Keycloak with a restart policy and writes its OPENSHELL_TEST_* variables,
so the image is cached with the disk. The runner no longer knows about
services: it passes every /etc/openshell-test/env.d/*.env variable to the
tests. Environment setup can now take inputs, for the realm file, and the
Keycloak fixture waits for the realm to answer after boot.

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

Author: matthewgrossman

tests/environments/docker.yaml

@@ -24,14 +24,14 @@ environments:
     tmachine: { environment: ubuntu-docker-rootful, installer: deb }
     gateway: openshell
 
-  # The same VM with Keycloak started after boot, for `features` suites that
-  # need an OIDC issuer. `services` adds `+<service>` to the ID; see
-  # tests/runner/src/services.rs.
+  # Ubuntu with Docker and Keycloak, for `features` suites that need an OIDC
+  # issuer. The tmachine environment runs Keycloak and publishes how to reach
+  # it; `services` only names it in the ID.
   - id: tmachine.ubuntu2404-amd64.docker-rootful.deb+keycloak
     backend: tmachine
     machine: { os: ubuntu, version: "24.04", arch: amd64 }
     runtime: docker-rootful
     installer: deb
-    tmachine: { environment: ubuntu-docker-rootful, installer: deb }
+    tmachine: { environment: ubuntu-docker-rootful-keycloak, installer: deb }
     gateway: openshell
     services: [keycloak]

tests/plans/README.md

@@ -19,23 +19,24 @@ replaced by `-`: `tests/file_transfer.rs` is the suite `file-transfer`.
 
 | Plan | Runs |
 |---|---|
-| `cpu-tests.yaml` | The `conformance` family on Ubuntu 24.04 amd64 with rootful Docker and the Debian package, and the `features` suites that need Keycloak on the same VM with Keycloak started |
+| `cpu-tests.yaml` | The `conformance` family on Ubuntu 24.04 amd64 with rootful Docker and the Debian package, and the `features` suites that need Keycloak on the same setup with Keycloak |
 
 ## Environment services
 
 A suite that needs something besides the gateway, such as an OIDC issuer, runs
-in an environment that lists it under `services`. The environment ID ends in
-`+<service>` for each one, e.g. `tmachine.ubuntu2404-amd64.docker-rootful.deb+keycloak`.
-
-After the VM boots, the runner applies
-`tests/tmachine/ansible/playbooks/services/<service>.yaml`. The playbook starts
-the service and writes `OPENSHELL_TEST_*` lines to
-`/etc/openshell-test/env.d/<service>.env`; the runner passes them to the tests.
-A suite-local fixture reads them, as `tests/suites/features/keycloak` does. To
+in an environment that provides it. tmachine declares that environment: its
+setup installs and starts the service (with a restart policy, since setup is
+cached as a disk image) and writes how to reach it as `OPENSHELL_TEST_*` lines
+in `/etc/openshell-test/env.d/<service>.env`. See the
+`ubuntu-docker-rootful-keycloak` environment in `tests/tmachine/config.nix`
+and `tests/tmachine/ansible/playbooks/keycloak.yaml`.
+
+The runner passes every variable in those files to the tests, and a
+suite-local fixture reads them, as `tests/suites/features/keycloak` does. To
 run such a suite against your own deployment, set the same variables yourself.
 
-To add a service, add it to `Service` in `tests/runner/src/services.rs`, write
-its playbook, and add an environment that lists it.
+In `tests/environments/`, list the service under `services`; the environment
+ID then ends in `+<service>`, e.g. `tmachine.ubuntu2404-amd64.docker-rootful.deb+keycloak`.
 
 ## How a plan runs on CI
 

tests/runner/src/environments.rs

@@ -17,8 +17,6 @@ use std::path::Path;
 use anyhow::{Context, Result, bail, ensure};
 use serde::Deserialize;
 
-use crate::services::Service;
-
 /// All environments, keyed by ID.
 #[derive(Debug, Default)]
 pub struct Catalog {
@@ -47,10 +45,11 @@ pub struct Environment {
     /// use the CLI's active gateway.
     #[serde(default)]
     pub gateway: Option<String>,
-    /// Services the runner starts after boot, such as Keycloak. See
-    /// [`crate::services`].
+    /// What the tmachine environment runs besides the runtime, such as
+    /// `keycloak`. Each adds `+<service>` to the ID, so plans can pick an
+    /// environment that has it. The environment, not the runner, starts them.
     #[serde(default)]
-    pub services: Vec<Service>,
+    pub services: Vec<String>,
 }
 
 #[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
@@ -126,7 +125,7 @@ impl Environment {
         );
         for service in &self.services {
             id.push('+');
-            id.push_str(&service.to_string());
+            id.push_str(service);
         }
         id
     }
@@ -262,7 +261,7 @@ environments:
         let env = environments
             .get("tmachine.ubuntu2404-amd64.docker-rootful.deb+keycloak")
             .unwrap();
-        assert_eq!(env.services, [Service::Keycloak]);
+        assert_eq!(env.services, ["keycloak"]);
 
         let without_suffix = yaml.replace("deb+keycloak\n", "deb\n");
         let err = catalog(&without_suffix).unwrap_err();

tests/runner/src/execute.rs

@@ -6,7 +6,9 @@
 //!
 //! Tests find the gateway the way the CLI does, so the runner points them at
 //! the one the installer registered with `OPENSHELL_GATEWAY` and at the
-//! installed CLI with `OPENSHELL_BIN`. There is no target file.
+//! installed CLI with `OPENSHELL_BIN`. There is no target file. Anything else
+//! the environment provides, such as Keycloak, the tests learn from the
+//! variables it publishes (see [`crate::test_env`]).
 //!
 //! Adapted from the executor in Evan Lezar's NVIDIA/OpenShell#4629.
 
@@ -22,7 +24,7 @@ use tmachine::config::Config;
 use crate::bundle::{Bundle, PROFILE, READY_BINARY};
 use crate::jobs::{FamilyJob, Job};
 use crate::report::{Outcome, Status, annotate_junit, verify_junit};
-use crate::services;
+use crate::test_env;
 
 const GUEST_ROOT: &str = "/var/tmp/openshell-tests";
 /// How long the gateway has to become healthy after the environment is up.
@@ -81,32 +83,6 @@ pub async fn run(job: &Job, options: &Options<'_>) -> Vec<Outcome> {
         }
     };
 
-    let service_env = if job.environment.services.is_empty() {
-        Ok(Vec::new())
-    } else {
-        eprintln!("==> {env_id}: starting services");
-        services::start(&session, &job.environment.services).await
-    };
-    let service_env = match service_env {
-        Ok(vars) => vars,
-        Err(error) => {
-            let message = format!("start services: {error:#}");
-            eprintln!("==> {env_id}: {message}");
-            if let Err(error) = session
-                .collect_diagnostics(&options.results.join(env_id).join("diagnostics"))
-                .await
-            {
-                eprintln!("==> {env_id}: collecting diagnostics failed: {error:#}");
-            }
-            session.down().await;
-            return job
-                .families
-                .iter()
-                .map(|family| outcome(family, Status::Error(message.clone())))
-                .collect();
-        }
-    };
-
     let mut outcomes = Vec::new();
     for family in &job.families {
         if !family.has_tests() {
@@ -119,7 +95,7 @@ pub async fn run(job: &Job, options: &Options<'_>) -> Vec<Outcome> {
             family.family,
             family.filterset()
         );
-        let status = match run_family(&session, job, family, &service_env, options, &dir).await {
+        let status = match run_family(&session, job, family, options, &dir).await {
             Ok(status) => status,
             Err(error) => Status::Error(format!("{error:#}")),
         };
@@ -176,14 +152,14 @@ async fn provision(job: &Job, config: &Config) -> Result<Session> {
 
 /// The environment variables that point the tests at the system under test:
 /// the registered gateway (when the environment names one), the installed
-/// CLI, and whatever the environment's services published.
-pub fn guest_env(job: &Job, openshell_bin: &str, service_env: &[String]) -> Vec<String> {
+/// CLI, and what the environment publishes for tests (see [`test_env`]).
+pub fn guest_env(job: &Job, openshell_bin: &str, test_env: &[String]) -> Vec<String> {
     let mut vars = Vec::new();
     if let Some(gateway) = &job.environment.gateway {
         vars.push(format!("OPENSHELL_GATEWAY={gateway}"));
     }
     vars.push(format!("OPENSHELL_BIN={openshell_bin}"));
-    vars.extend(service_env.iter().cloned());
+    vars.extend(test_env.iter().cloned());
     vars
 }
 
@@ -236,7 +212,6 @@ async fn run_family(
     session: &Session,
     job: &Job,
     family: &FamilyJob,
-    service_env: &[String],
     options: &Options<'_>,
     dir: &Path,
 ) -> Result<Status> {
@@ -258,7 +233,10 @@ async fn run_family(
         !openshell_bin.is_empty(),
         "the installer did not put openshell on the guest's PATH"
     );
-    let env = guest_env(job, openshell_bin, service_env);
+    let test_env = test_env::read(session)
+        .await
+        .context("read the variables the environment publishes for tests")?;
+    let env = guest_env(job, openshell_bin, &test_env);
 
     let guest = format!("{GUEST_ROOT}/{}", family.family);
     checked(

tests/runner/src/lib.rs

@@ -13,4 +13,4 @@ pub mod github;
 pub mod jobs;
 pub mod plan;
 pub mod report;
-pub mod services;
+pub mod test_env;