feat(test-guest): add snap lifecycle reproduction harness (#2865)

* feat(test-guest): add snap lifecycle reproduction harness

Signed-off-by: Evan Lezar <elezar@nvidia.com>

* fix(test-guest): preserve cache seal executability

Signed-off-by: Evan Lezar <elezar@nvidia.com>

---------

Signed-off-by: Evan Lezar <elezar@nvidia.com>

Author: elezar

nix/test-guest/README.md

@@ -63,6 +63,11 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-gues
 | Fedora 44 | No | Yes | Yes | `.rpm` |
 | Rocky Linux 9 | Yes | Yes | Yes | `.rpm` |
 
+The `snapd` configuration is available for Ubuntu and prepares snapd for
+local Snap lifecycle experiments. It does not install Docker, because the Snap
+gateway reproduction uses the Docker **Snap** and its `docker:docker-daemon`
+interface rather than the host-package Docker configuration.
+
 The Ubuntu 24.04 Podman configuration is available for runtime and packaging
 checks, but its Podman 4 release does not provide the `pasta` rootless network
 helper required by OpenShell sandbox callbacks. OpenShell Podman E2E runs use
@@ -121,9 +126,11 @@ nix run .#test-guest -- \
 
 Configurations are Ansible playbooks stored under `nix/test-guest/configuration/`. Ansible runs on the host using the VM's ephemeral SSH key and loopback port. The guest does not install Ansible.
 
-Configurations run in the order provided on the command line. OpenShell packages and copied binaries are installed after all configurations succeed.
+Configurations run in the order provided on the command line. OpenShell packages and copied files are installed after all configurations succeed.
 
-`--install` packages and `--copy` executables are applied by a dedicated per-run Ansible playbook. They are not stored in prepared VM cache entries.
+`--install` packages and `--copy` files are applied by a dedicated per-run
+Ansible playbook. `--copy` preserves each source file's ordinary permission
+bits. They are not stored in prepared VM cache entries.
 
 ## Prepared VM cache
 
@@ -212,9 +219,10 @@ For an x86_64 Linux guest, supply x86_64 binaries and use `package:deb:amd64`. T
 
 `--install` is repeatable. Debian packages are accepted by Ubuntu; RPM packages are accepted by CentOS, Fedora, and Rocky Linux. This prototype can install an existing RPM but does not build one.
 
-## Copy binaries directly
+## Copy files directly
 
-Use `--copy SOURCE:DEST` to install an executable without creating a package:
+Use `--copy SOURCE:DEST` to copy a regular file without creating a package. The
+guest file preserves the source's ordinary permission bits:
 
 ```shell
 nix run .#test-guest -- \
@@ -223,6 +231,29 @@ nix run .#test-guest -- \
   -- openshell --version
 ```
 
+## Reproduce Snap gateway startup
+
+The gateway Snap must be native to the guest architecture. Copy an existing
+Snap artifact and the reproduction script into a prepared Ubuntu guest, then
+run the script as root. It follows the Release Canary ordering exactly: install
+the Snap, connect Docker/log/system interfaces, and immediately query the
+gateway. On each failure it prints snapd and gateway journals.
+
+```shell
+nix run .#test-guest -- \
+  --distro ubuntu \
+  --with snapd \
+  --keep \
+  --copy ./openshell_*.snap:/tmp/openshell.snap \
+  --copy ./nix/test-guest/scripts/snap-gateway-repro.sh:/usr/local/bin/snap-gateway-repro \
+  -- sudo /usr/local/bin/snap-gateway-repro /tmp/openshell.snap 10 30
+```
+
+`--keep` retains the overlay and serial log when diagnosing a failure. The
+runner prints their location after shutdown. The final `30` accepts automatic
+recovery for up to 30 seconds; omit it to require the canary's immediate check.
+
+
 The destination must be an absolute guest path. Copied files are installed with mode `0755`.
 
 ## Runner options
@@ -231,7 +262,8 @@ The destination must be an absolute guest path. Copied files are installed with
 --distro NAME       Base distro: ubuntu, centos, fedora, or rocky
 --with NAME         Apply docker, podman, or selinux; repeatable
 --install PATH      Install a .deb or .rpm package; repeatable
---copy SRC:DEST     Copy an executable into the guest; repeatable
+--copy SRC:DEST     Copy a regular file into the guest, preserving its host mode;
+                    repeatable
 --ssh-port PORT     Use a specific loopback SSH forwarding port
 --forward-port HOST_PORT:GUEST_PORT
                     Forward a loopback host port to a guest port; repeatable

nix/test-guest/cache-seal.sh

@@ -0,0 +1,38 @@
+---
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+- name: Configure snapd
+  hosts: test_vm
+  become: true
+  gather_facts: true
+
+  tasks:
+    - name: Validate snapd support
+      ansible.builtin.assert:
+        that:
+          - ansible_facts.distribution == "Ubuntu"
+        fail_msg: >-
+          snapd is currently configured only for Ubuntu test guests,
+          not {{ ansible_facts.distribution }}.
+
+    - name: Refresh Ubuntu package metadata
+      ansible.builtin.apt:
+        update_cache: true
+
+    - name: Install snapd
+      ansible.builtin.apt:
+        name: snapd
+        state: present
+        install_recommends: false
+
+    - name: Start snapd socket activation
+      ansible.builtin.systemd_service:
+        name: snapd.socket
+        enabled: true
+        state: started
+
+    - name: Wait for snapd seed
+      ansible.builtin.command:
+        cmd: snap wait system seed.loaded
+      changed_when: false

nix/test-guest/configuration/snapd.yml

@@ -28,6 +28,7 @@ let
     docker = ./configuration/docker.yml;
     podman = ./configuration/podman.yml;
     selinux = ./configuration/selinux.yml;
+    snapd = ./configuration/snapd.yml;
   };
 
   mkDistroProfile =

nix/test-guest/default.nix

@@ -13,9 +13,10 @@ Usage:
 
 Options:
   --distro NAME       Base distro: ubuntu, centos, fedora, or rocky
-  --with NAME         Apply a configuration; repeatable (docker, podman, selinux)
+  --with NAME         Apply a configuration; repeatable (docker, podman, selinux, snapd)
   --install PATH      Install a .deb or .rpm package; repeatable
-  --copy SRC:DEST     Copy an executable to an absolute guest path; repeatable
+  --copy SRC:DEST     Copy a regular file to an absolute guest path, preserving
+                      its host mode; repeatable
   --ssh-port PORT     Use a specific loopback SSH forwarding port
   --forward-port HOST_PORT:GUEST_PORT
                       Forward a loopback host port to a guest port; repeatable
@@ -43,6 +44,26 @@ require_value() {
 	fi
 }
 
+preserved_file_mode() {
+	local source_path=$1
+	local source_mode
+
+	if [ "$(uname -s)" = Darwin ]; then
+		if ! source_mode=$(stat -f '%Lp' "${source_path}"); then
+			echo "could not determine mode for --copy source: ${source_path}" >&2
+			return 1
+		fi
+	elif ! source_mode=$(stat -c '%a' "${source_path}"); then
+		echo "could not determine mode for --copy source: ${source_path}" >&2
+		return 1
+	fi
+	if [[ ! ${source_mode} =~ ^[0-7]{3,4}$ ]]; then
+		echo "could not determine mode for --copy source: ${source_path}" >&2
+		return 1
+	fi
+	printf '%03o\n' "$((8#${source_mode} & 8#777))"
+}
+
 distro=
 requested_ssh_port=
 keep=0
@@ -618,13 +639,14 @@ if [ "${#packages[@]}" -gt 0 ] || [ "${#copies[@]}" -gt 0 ]; then
 	for copy_spec in "${copies[@]}"; do
 		source_path=${copy_spec%%:*}
 		destination=${copy_spec#*:}
+		mode=$(preserved_file_mode "${source_path}") || exit 2
 		remote_path=${artifact_staging_dir}/copy-${artifact_index}
 		echo "==> Copying artifact: ${destination}"
 		scp -q "${scp_args[@]}" \
 			"${source_path}" "openshell@127.0.0.1:${remote_path}"
 		printf -v install_command \
-			'sudo install -D -m 0755 -- %q %q' \
-			"${remote_path}" "${destination}"
+			'sudo install -D -m %q -- %q %q' \
+			"${mode}" "${remote_path}" "${destination}"
 		ssh "${ssh_args[@]}" openshell@127.0.0.1 "${install_command}"
 		artifact_index=$((artifact_index + 1))
 	done

nix/test-guest/run.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Reproduce the Release Canary Snap lifecycle: install the OpenShell Snap,
+# connect its interfaces after the daemon is started, then immediately use the
+# local gateway. Run this as root inside an Ubuntu guest prepared with --with snapd.
+
+set -uo pipefail
+
+usage() {
+	cat <<'EOF'
+Usage: snap-gateway-repro.sh SNAP_FILE [ATTEMPTS] [READY_TIMEOUT_SECONDS]
+
+Install SNAP_FILE repeatedly using the Release Canary interface ordering.
+ATTEMPTS defaults to 1. READY_TIMEOUT_SECONDS defaults to 0, preserving the
+canary's immediate readiness check. Set it to a positive value to wait for
+automatic gateway recovery after the immediate check fails. Every failed
+attempt prints service, connection, snap-change, journal, gateway-log, and
+listener diagnostics.
+EOF
+}
+
+if [ "$#" -lt 1 ] || [ "$#" -gt 3 ]; then
+	usage >&2
+	exit 2
+fi
+
+snap_file=$1
+attempts=${2:-1}
+ready_timeout=${3:-0}
+if [ ! -f "${snap_file}" ]; then
+	echo "Snap file does not exist: ${snap_file}" >&2
+	exit 2
+fi
+if [[ ! ${attempts} =~ ^[1-9][0-9]*$ ]]; then
+	echo "ATTEMPTS must be a positive integer: ${attempts}" >&2
+	exit 2
+fi
+if [[ ! ${ready_timeout} =~ ^[0-9]+$ ]]; then
+	echo "READY_TIMEOUT_SECONDS must be a non-negative integer: ${ready_timeout}" >&2
+	exit 2
+fi
+
+diagnostics() {
+	local attempt=$1
+	echo "========== Snap diagnostics (attempt ${attempt}) ==========" >&2
+	snap services openshell >&2 || true
+	snap connections openshell >&2 || true
+	snap changes >&2 || true
+	systemctl status snap.openshell.gateway.service --no-pager >&2 || true
+	journalctl -b -u snap.openshell.gateway.service --no-pager -n 300 >&2 || true
+	journalctl -b -u snapd.service --no-pager -n 300 >&2 || true
+	snap logs openshell.gateway -n=300 >&2 || true
+	ss -ltnp '( sport = :17670 )' >&2 || true
+}
+
+gateway_is_ready() {
+	runuser -u openshell -- /snap/bin/openshell status >/dev/null 2>&1
+}
+
+wait_for_gateway() {
+	local deadline=$((SECONDS + ready_timeout))
+	while [ "${SECONDS}" -lt "${deadline}" ]; do
+		if gateway_is_ready; then
+			return 0
+		fi
+		sleep 1
+	done
+	gateway_is_ready
+}
+
+if ! snap list docker >/dev/null 2>&1; then
+	echo "==> Installing Docker Snap"
+	snap install docker
+fi
+
+failures=0
+for attempt in $(seq 1 "${attempts}"); do
+	echo "==> Snap gateway reproduction attempt ${attempt}/${attempts}"
+	snap remove --purge openshell >/dev/null 2>&1 || true
+	rm -rf /home/openshell/snap/openshell
+
+	if ! snap install "${snap_file}" --dangerous ||
+		! snap connect openshell:docker docker:docker-daemon ||
+		! snap connect openshell:log-observe ||
+		! snap connect openshell:system-observe; then
+		echo "OpenShell installation or interface connection failed" >&2
+		diagnostics "${attempt}"
+		failures=$((failures + 1))
+		continue
+	fi
+
+	# This deliberately does not wait for the listener. It mirrors the canary
+	# and exposes a daemon that fails or races after late interface connections.
+	if ! runuser -u openshell -- /snap/bin/openshell gateway add \
+		http://127.0.0.1:17670 --local --name snap-docker ||
+		! runuser -u openshell -- /snap/bin/openshell gateway select snap-docker ||
+		! gateway_is_ready; then
+		if [ "${ready_timeout}" -gt 0 ] && wait_for_gateway; then
+			echo "Gateway recovered automatically within ${ready_timeout}s"
+			continue
+		fi
+		echo "Gateway was not usable immediately after interface connection" >&2
+		diagnostics "${attempt}"
+		failures=$((failures + 1))
+	fi
+done
+
+if [ "${failures}" -gt 0 ]; then
+	echo "${failures}/${attempts} attempt(s) failed" >&2
+	exit 1
+fi
+
+echo "All ${attempts} attempt(s) passed"