fix(ci): restore default e2e coverage with a Kubernetes recovery waiver (#4424)

* fix(ci): make conformance-only e2e selection explicit

Signed-off-by: Evan Lezar <elezar@nvidia.com>

* fix(ci): use mapped NodePort for kind gateway sessions

Signed-off-by: Evan Lezar <elezar@nvidia.com>

* fix(ci): waive SSH transport recovery only on Kubernetes

Signed-off-by: Evan Lezar <elezar@nvidia.com>

* fix(e2e): log a notice when runners skip Rust e2e tests

Emit a GitHub Actions notice from the Docker, Podman, Kubernetes, and VM
runners when e2e-conformance-only skips the Rust tests, so a narrowed lane
is visible in the job summary instead of silently reporting success.

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

* docs(testing): move e2e conformance-only selector to shared section

The selector applies to all four driver runners, so document it with the
Rust CLI e2e suites instead of under the Kubernetes task variants.

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

---------

Signed-off-by: Evan Lezar <elezar@nvidia.com>
Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
Co-authored-by: Matthew Grossman <mgrossman@nvidia.com>

Author: elezar

.config/nextest.toml

@@ -10,6 +10,10 @@ failure-output = "immediate-final"
 slow-timeout = { period = "30s", terminate-after = 4 }
 
 [profile.e2e-kubernetes]
+# Temporary waiver: port-forward reconnect can attach read-only while the old
+# SSH session retains stdin ownership. Keep recovery coverage on other drivers.
+# Remove this exclusion after resolving https://github.com/NVIDIA/OpenShell/issues/4445.
+default-filter = "not test(/^canonical_main_connect_recovers_its_ssh_transport$/)"
 fail-fast = false
 status-level = "slow"
 final-status-level = "fail"

.github/workflows/e2e-kubernetes-test.yml

@@ -48,7 +48,7 @@ on:
         type: string
         default: ""
       kubernetes-features:
-        description: "Cargo feature list for the Kubernetes e2e crate (uses script defaults when empty)"
+        description: "Cargo feature list for Kubernetes e2e (defaults when empty; e2e-conformance-only skips Rust tests)"
         required: false
         type: string
         default: ""

TESTING.md

@@ -249,6 +249,18 @@ Suites:
 - VM suite (`--features e2e-vm`) - runs e2e tests on a VM.
 - Kubernetes credential-driver suite (`--features e2e-kubernetes-credential-drivers`) - targeted Kubernetes Secrets and Vault provider credential storage coverage.
 
+The Docker, Podman, Kubernetes, and VM runners execute CLI
+conformance before their Rust tests. Set the driver's feature variable
+(`OPENSHELL_E2E_DOCKER_FEATURES`, `OPENSHELL_E2E_PODMAN_FEATURES`,
+`OPENSHELL_E2E_KUBERNETES_FEATURES`, or `OPENSHELL_E2E_VM_FEATURES`) to
+`e2e-conformance-only` to skip the Rust tests; the runner logs a notice when
+it skips them. Unset or empty values select the driver's default Rust
+features. The selector is handled by the runner; it is not a Cargo feature.
+
+```shell
+OPENSHELL_E2E_KUBERNETES_FEATURES=e2e-conformance-only mise run e2e:kubernetes
+```
+
 GPU device-selection tests compare OpenShell sandboxes against a plain Docker or
 Podman container that requests `--device nvidia.com/gpu=all`. The probe image
 defaults to the image used by the `gateway` stage in
@@ -389,6 +401,12 @@ mise run e2e:kubernetes:credential-drivers
 
 ### Kubernetes E2E (`e2e/rust/e2e-kubernetes.sh`)
 
+The `e2e-kubernetes` nextest profile temporarily excludes only
+`canonical_main_connect_recovers_its_ssh_transport` while
+[#4445](https://github.com/NVIDIA/OpenShell/issues/4445) investigates delayed SSH
+input-owner release through port-forwarding. The test remains available to other
+driver profiles; the remaining Kubernetes tests stay enabled.
+
 Kubernetes e2e tests deploy an OpenShell gateway into a real Kubernetes cluster
 via Helm and run the Rust e2e suite against it. On vanilla Kubernetes the harness
 reaches the gateway through `kubectl port-forward`; on OpenShift it instead uses a

e2e/rust/e2e-docker.sh

@@ -11,7 +11,8 @@ set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 E2E_TEST="${OPENSHELL_E2E_DOCKER_TEST:-}"
-E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES-e2e-docker}"
+# Unset or empty selects Rust defaults; e2e-conformance-only skips Rust tests.
+E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e-docker}"
 DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml"
 RUN_WITH_GATEWAY_COMMAND="__openshell_run_docker_e2e"
 # shellcheck source=e2e/support/conformance.sh
@@ -20,7 +21,8 @@ source "${ROOT}/e2e/support/conformance.sh"
 run_e2e() {
   e2e_run_openshell_conformance "Docker"
 
-  if [ -z "${E2E_FEATURES}" ]; then
+  if [ "${E2E_FEATURES}" = "e2e-conformance-only" ]; then
+    echo "::notice::Skipping Rust e2e tests (e2e-conformance-only)"
     return 0
   fi
 
@@ -44,7 +46,7 @@ if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then
   exit 0
 fi
 
-if [ -n "${E2E_FEATURES}" ]; then
+if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then
   CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh"
 fi
 

e2e/rust/e2e-kubernetes.sh

@@ -26,7 +26,8 @@ RUN_WITH_GATEWAY_COMMAND="__openshell_run_kubernetes_e2e"
 # shellcheck source=e2e/support/conformance.sh
 source "${ROOT}/e2e/support/conformance.sh"
 
-E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES-e2e,e2e-host-gateway,e2e-kubernetes}"
+# Unset or empty selects Rust defaults; e2e-conformance-only skips Rust tests.
+E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES:-e2e,e2e-host-gateway,e2e-kubernetes}"
 
 # Fixed output path of the `e2e-kubernetes` nextest profile (`.config/nextest.toml`).
 JUNIT_XML="${ROOT}/results/e2e-kubernetes.xml"
@@ -90,7 +91,8 @@ run_suite() {
 
 run_e2e() {
   run_conformance
-  if [ -z "${E2E_FEATURES}" ]; then
+  if [ "${E2E_FEATURES}" = "e2e-conformance-only" ]; then
+    echo "::notice::Skipping Rust e2e tests (e2e-conformance-only)"
     return 0
   fi
 

e2e/rust/e2e-podman.sh

@@ -11,7 +11,8 @@ set -euo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 E2E_TEST="${OPENSHELL_E2E_PODMAN_TEST:-}"
 E2E_TEST_SET="${OPENSHELL_E2E_PODMAN_TEST_SET:-}"
-E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES-e2e-podman}"
+# Unset or empty selects Rust defaults; e2e-conformance-only skips Rust tests.
+E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman}"
 DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml"
 RUN_WITH_GATEWAY_COMMAND="__openshell_run_podman_e2e"
 # shellcheck source=e2e/support/conformance.sh
@@ -72,7 +73,8 @@ fi
 
 if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then
   e2e_run_openshell_conformance "Podman"
-  if [ -z "${E2E_FEATURES}" ]; then
+  if [ "${E2E_FEATURES}" = "e2e-conformance-only" ]; then
+    echo "::notice::Skipping Rust e2e tests (e2e-conformance-only)"
     exit 0
   fi
 
@@ -98,7 +100,7 @@ if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ] && [ -z "${OPENSHELL_BIN:-}" ]; then
   export OPENSHELL_BIN="${ROOT}/target/debug/openshell"
 fi
 
-if [ -n "${E2E_FEATURES}" ]; then
+if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then
   CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh"
 fi
 

e2e/rust/e2e-vm.sh

@@ -50,7 +50,8 @@ GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}"
 DRIVER_BIN="${OPENSHELL_VM_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm}"
 CLI_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}"
 E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}"
-E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES-e2e-vm}"
+# Unset or empty selects Rust defaults; e2e-conformance-only skips Rust tests.
+E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}"
 SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-nvcr.io/nvidia/base/ubuntu:24.04}"
 
 # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`.
@@ -405,6 +406,10 @@ e2e_export_gateway_restart_metadata \
 export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}"
 
 e2e_run_openshell_conformance "VM"
+if [ "${E2E_FEATURES}" = "e2e-conformance-only" ]; then
+  echo "::notice::Skipping Rust e2e tests (e2e-conformance-only)"
+  exit 0
+fi
 
 # Seed the catalog once for the whole lane. The profiles live in the gateway
 # for as long as it runs, and the import is create-only: a second import of

tasks/test.toml

@@ -273,12 +273,12 @@ run = "bash e2e/no-compute-driver-gateway.sh"
 
 ["e2e:docker:external-driver"]
 description = "Run Docker conformance with a driver-free gateway and external Docker driver binary"
-env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_DOCKER_FEATURES = "" }
+env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-conformance-only" }
 run = "e2e/rust/e2e-docker.sh"
 
 ["e2e:podman:external-driver"]
 description = "Run Podman conformance with a driver-free gateway and external Podman driver binary"
-env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_FEATURES = "" }
+env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_FEATURES = "e2e-conformance-only" }
 run = "e2e/rust/e2e-podman.sh"
 
 ["e2e:vm:external-driver"]
@@ -288,7 +288,7 @@ run = "e2e/rust/e2e-vm.sh"
 
 ["e2e:kubernetes:external-driver"]
 description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver"
-env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" }
+env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e-conformance-only" }
 run = "e2e/rust/e2e-kubernetes.sh"
 
 ["e2e:docker"]