From 4b49638d4df595e095eb94f5c3aab238996042bb Mon Sep 17 00:00:00 2001 From: Vineeth Sai Narajala Date: Mon, 2 Nov 2026 20:28:08 +0000 Subject: [PATCH 1/2] feat(vm): add local image cache inspection and pruning Signed-off-by: Vineeth Sai Narajala --- crates/openshell-driver-vm/README.md | 31 ++ crates/openshell-driver-vm/src/driver.rs | 57 +- .../src/image_cache_admin.rs | 513 ++++++++++++++++++ crates/openshell-driver-vm/src/main.rs | 132 ++++- docs/how-it-works/sandboxes/runtimes.mdx | 14 + skills/debug-openshell-cluster/SKILL.md | 7 + 6 files changed, 746 insertions(+), 8 deletions(-) create mode 100644 crates/openshell-driver-vm/src/image_cache_admin.rs diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index a0371bacfb..a379e9f819 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -239,6 +239,37 @@ Image and writable-overlay preparation run in owned worker processes. Stopping o At startup, the driver reclaims inactive attempts in this directory. It preserves active attempts, committed image caches, and unmarked staging from older releases. Concurrent preparations serialize image-cache publication; an interrupted attempt cannot publish a partial disk over an existing cache entry. Gateway upload slots for `rootfs_tar_path` keep their existing, separate lifecycle. +### Inspect and prune image disks + +Run the VM driver binary on the gateway host with the same `state_dir` configured +for the gateway: + +```shell +openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache list +openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache prune --unused --dry-run +``` + +`list` reports each committed root disk's cache ID, image reference, allocated +and apparent bytes, last-use Unix timestamp in milliseconds, cache layout +version, and persisted sandbox IDs that reference it. Older entries without +cache metadata show `-` for the image reference and last use. New cache uses +record this metadata without changing the disk contents. + +Stop the VM gateway before removing entries, then run: + +```shell +openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache prune --unused +``` + +Prune removes only committed disks that no persisted sandbox references. It +leaves disks referenced by stopped sandboxes in place. For sandboxes created +before bootstrap identities were persisted, it conservatively preserves all +bootstrap disks. Prune refuses to run while a current driver holds the state +lock or an older driver is reachable at the default state socket. The reported +bytes are the disks' allocated bytes before removal; filesystem snapshots and +shared blocks can make actual free space differ. Nothing is removed +automatically. + Each sandbox gets its own sparse writable `/sandboxes//overlay.ext4`. Guest init mounts overlayfs as `/` with the prepared image rootfs as lowerdir when present, otherwise the bootstrap diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 6260a52842..9d4f399bd2 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -6,6 +6,9 @@ #[path = "preparation.rs"] mod preparation; +#[path = "image_cache_admin.rs"] +pub mod image_cache_admin; + #[cfg(test)] #[path = "registry_cache_tests.rs"] mod registry_cache_tests; @@ -1897,6 +1900,20 @@ impl VmDriver { .await? }; let image_identity = image_plan.image_identity.clone(); + let bootstrap_image_ref = self.bootstrap_image_ref()?; + if let Err(error) = image_cache_admin::record_use( + &self.config.state_dir, + &image_plan.bootstrap_image_identity, + &bootstrap_image_ref, + ) { + warn!(%error, "vm driver: could not record bootstrap cache use"); + } + if image_plan.image_disk.is_some() + && let Err(error) = + image_cache_admin::record_use(&self.config.state_dir, &image_identity, &image_ref) + { + warn!(%error, "vm driver: could not record prepared image cache use"); + } self.ensure_provisioning_active(&sandbox.id).await?; info!( sandbox_id = %sandbox.id, @@ -1946,8 +1963,13 @@ impl VmDriver { .map_err(|err| Status::internal(format!("prepare guest overlay disk failed: {err}")))?; self.ensure_provisioning_active(&sandbox.id).await?; - if let Err(err) = - write_sandbox_image_metadata(&state_dir, &image_ref, &image_identity).await + if let Err(err) = write_sandbox_image_metadata( + &state_dir, + &image_ref, + &image_identity, + &image_plan.bootstrap_image_identity, + ) + .await { return Err(Status::internal(format!( "write sandbox image metadata failed: {err}" @@ -5225,6 +5247,11 @@ impl VmDriver { #[doc(hidden)] pub async fn run_image_preparation_worker(request_path: &Path) -> Result<(), String> { let (request, directory) = preparation::read_request(request_path)?; + let _cache_lease = image_cache_admin::lock_cache( + &request.config.state_dir, + image_cache_admin::LockMode::Shared, + ) + .map_err(|error| format!("lock image cache for preparation: {error}"))?; validate_sandbox_id(&request.sandbox_id).map_err(|error| error.message().to_string())?; preparation::watch_parent(request.parent_pid)?; tracing_subscriber::fmt() @@ -7365,6 +7392,7 @@ async fn write_sandbox_image_metadata( state_dir: &Path, image_ref: &str, image_identity: &str, + bootstrap_image_identity: &str, ) -> Result<(), std::io::Error> { tokio::fs::write( state_dir.join(IMAGE_IDENTITY_FILE), @@ -7376,6 +7404,14 @@ async fn write_sandbox_image_metadata( format!("{image_ref}\n"), ) .await?; + if let Err(error) = tokio::fs::write( + state_dir.join(image_cache_admin::BOOTSTRAP_IDENTITY_FILE), + format!("{bootstrap_image_identity}\n"), + ) + .await + { + warn!(%error, "vm driver: could not persist bootstrap cache identity"); + } Ok(()) } @@ -9163,9 +9199,20 @@ mod tests { ) .await .unwrap(); - write_sandbox_image_metadata(&state_dir, "unused-image", "sha256:original-image") - .await - .unwrap(); + write_sandbox_image_metadata( + &state_dir, + "unused-image", + "sha256:original-image", + "sha256:bootstrap-image", + ) + .await + .unwrap(); + assert_eq!( + tokio::fs::read_to_string(state_dir.join(image_cache_admin::BOOTSTRAP_IDENTITY_FILE)) + .await + .unwrap(), + "sha256:bootstrap-image\n" + ); tokio::fs::write(state_dir.join(marker), b"inactive\n") .await .unwrap(); diff --git a/crates/openshell-driver-vm/src/image_cache_admin.rs b/crates/openshell-driver-vm/src/image_cache_admin.rs new file mode 100644 index 0000000000..ab1e6688de --- /dev/null +++ b/crates/openshell-driver-vm/src/image_cache_admin.rs @@ -0,0 +1,513 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Local, operator-initiated inspection and cleanup of VM root-disk caches. + +#![allow(unsafe_code)] + +use super::{ + IMAGE_CACHE_ROOTFS_IMAGE, IMAGE_IDENTITY_FILE, SANDBOX_REQUEST_FILE, image_cache_dir, + image_cache_root_dir, sandboxes_root_dir, sanitize_image_identity, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const LOCK_FILE: &str = ".image-cache.lock"; +const METADATA_FILE: &str = "cache-metadata.json"; +pub(super) const BOOTSTRAP_IDENTITY_FILE: &str = "bootstrap-image-identity"; +const MAX_METADATA_BYTES: u64 = 16 * 1024; +static METADATA_WRITE_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum LockMode { + Shared, + Exclusive, +} + +pub struct CacheLease { + _file: File, + state_dir: PathBuf, + mode: LockMode, +} + +impl CacheLease { + pub fn list(&self) -> io::Result> { + list(&self.state_dir) + } + + pub fn prune_unused(&self, dry_run: bool) -> io::Result { + if !dry_run { + if self.mode != LockMode::Exclusive { + return Err(io::Error::new( + io::ErrorKind::WouldBlock, + "prune requires an exclusive image cache lease", + )); + } + refuse_live_legacy_driver(&self.state_dir)?; + } + prune_unused(&self.state_dir, dry_run) + } +} + +/// A shared lease covers the driver and its image workers. Prune takes the +/// exclusive lease, so it cannot remove a disk being prepared or attached. +pub fn lock_cache(state_dir: &Path, mode: LockMode) -> io::Result { + fs::create_dir_all(state_dir)?; + let state_dir = fs::canonicalize(state_dir)?; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(state_dir.join(LOCK_FILE))?; + if !file.metadata()?.is_file() { + return Err(io::Error::other("image cache lock is not a regular file")); + } + let operation = match mode { + LockMode::Shared => libc::LOCK_SH, + LockMode::Exclusive => libc::LOCK_EX, + } | libc::LOCK_NB; + let deadline = Instant::now() + Duration::from_millis(500); + loop { + // SAFETY: flock receives a valid borrowed file descriptor and no pointers. + if unsafe { libc::flock(file.as_raw_fd(), operation) } == 0 { + return Ok(CacheLease { + _file: file, + state_dir, + mode, + }); + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::WouldBlock { + return Err(error); + } + if mode == LockMode::Shared || Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::WouldBlock, + "image cache is busy; stop the VM gateway before pruning", + )); + } + // A process starting during a read can inherit the shared descriptor + // briefly. Wait for that transient lease before reporting a live driver. + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// Older driver binaries do not hold the cache lease. Detect the default +/// gateway socket before deleting anything in a state dir they may be using. +fn refuse_live_legacy_driver(state_dir: &Path) -> io::Result<()> { + let socket = state_dir.join("run").join("compute-driver.sock"); + match UnixStream::connect(socket) { + Ok(_) => Err(io::Error::new( + io::ErrorKind::WouldBlock, + "VM gateway is running; stop it before pruning", + )), + Err(error) + if matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused + ) => + { + Ok(()) + } + Err(error) => Err(error), + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CacheMetadata { + image_ref: String, + last_used_unix_ms: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheEntry { + pub id: String, + pub image_ref: Option, + pub allocated_bytes: u64, + pub apparent_bytes: u64, + pub last_used_unix_ms: Option, + pub cache_version: String, + pub sandboxes: Vec, +} + +#[derive(Debug, Default)] +pub struct PruneReport { + pub entries: Vec, + pub allocated_bytes: u64, +} + +/// Metadata is best effort: a full disk must not prevent an otherwise ready +/// sandbox from starting. It is never used to decide whether pruning is safe. +pub fn record_use(state_dir: &Path, id: &str, image_ref: &str) -> io::Result<()> { + let dir = image_cache_dir(state_dir, id); + require_regular_file(&dir.join(IMAGE_CACHE_ROOTFS_IMAGE))?; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(io::Error::other)?; + let metadata = CacheMetadata { + image_ref: image_ref.to_owned(), + last_used_unix_ms: u64::try_from(timestamp.as_millis()).map_err(io::Error::other)?, + }; + let sequence = METADATA_WRITE_COUNTER.fetch_add(1, Ordering::Relaxed); + let temporary = dir.join(format!( + ".{METADATA_FILE}.{}.{sequence}.tmp", + std::process::id() + )); + let destination = dir.join(METADATA_FILE); + let result = (|| { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + serde_json::to_writer(&mut file, &metadata)?; + file.write_all(b"\n")?; + file.sync_all()?; + fs::rename(&temporary, destination) + })(); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} + +fn list(state_dir: &Path) -> io::Result> { + let references = sandbox_references(state_dir)?; + let root = image_cache_root_dir(state_dir); + if !directory_exists(&root)? { + return Ok(Vec::new()); + } + let mut entries = Vec::new(); + for item in fs::read_dir(root)? { + let item = item?; + let id = item.file_name().to_string_lossy().into_owned(); + if !id.starts_with("sandbox-") || id.contains(".staging-") { + continue; + } + if !item.file_type()?.is_dir() { + return Err(io::Error::other(format!( + "cache entry {id} is not a directory" + ))); + } + let disk = item.path().join(IMAGE_CACHE_ROOTFS_IMAGE); + let disk_metadata = match fs::symlink_metadata(&disk) { + Ok(metadata) if metadata.is_file() => metadata, + Ok(_) => { + return Err(io::Error::other(format!( + "cache disk for {id} is not a regular file" + ))); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + let metadata = read_metadata(&item.path().join(METADATA_FILE))?; + let mut sandboxes = references.by_id.get(&id).cloned().unwrap_or_default(); + if id.starts_with("sandbox-bootstrap-") { + sandboxes.extend(references.legacy_bootstrap.iter().cloned()); + } + sandboxes.sort(); + sandboxes.dedup(); + entries.push(CacheEntry { + cache_version: id + .split_once("-openshell-") + .map_or_else(|| "unknown".to_owned(), |(version, _)| version.to_owned()), + id, + image_ref: metadata.as_ref().map(|value| value.image_ref.clone()), + allocated_bytes: disk_metadata.blocks().saturating_mul(512), + apparent_bytes: disk_metadata.len(), + last_used_unix_ms: metadata.map(|value| value.last_used_unix_ms), + sandboxes, + }); + } + entries.sort_by(|left, right| left.id.cmp(&right.id)); + Ok(entries) +} + +/// The caller must hold an exclusive cache lease for a real prune. +fn prune_unused(state_dir: &Path, dry_run: bool) -> io::Result { + let mut report = PruneReport::default(); + for entry in list(state_dir)? { + if !entry.sandboxes.is_empty() { + continue; + } + if !dry_run { + fs::remove_dir_all(image_cache_root_dir(state_dir).join(&entry.id))?; + } + report.allocated_bytes = report.allocated_bytes.saturating_add(entry.allocated_bytes); + report.entries.push(entry); + } + Ok(report) +} + +fn read_metadata(path: &Path) -> io::Result> { + if !regular_file_exists(path)? { + return Ok(None); + } + let file = match File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + if file.metadata()?.len() > MAX_METADATA_BYTES { + return Ok(None); + } + Ok(serde_json::from_reader(file).ok()) +} + +#[derive(Default)] +struct SandboxReferences { + by_id: HashMap>, + legacy_bootstrap: Vec, +} + +fn sandbox_references(state_dir: &Path) -> io::Result { + let root = sandboxes_root_dir(state_dir); + if !directory_exists(&root)? { + return Ok(SandboxReferences::default()); + } + let mut references: HashMap> = HashMap::new(); + let mut legacy_bootstrap = Vec::new(); + for item in fs::read_dir(root)? { + let item = item?; + if !item.file_type()?.is_dir() { + return Err(io::Error::other( + "sandbox state contains a non-directory entry", + )); + } + let sandbox_id = item.file_name().to_string_lossy().into_owned(); + let path = item.path(); + let request = path.join(SANDBOX_REQUEST_FILE); + let image_identity = path.join(IMAGE_IDENTITY_FILE); + let request_exists = regular_file_exists(&request)?; + let image_exists = regular_file_exists(&image_identity)?; + if !request_exists && !image_exists { + continue; + } + if !image_exists { + return Err(io::Error::other(format!( + "sandbox {sandbox_id} has no image identity; cache prune is unsafe" + ))); + } + let image_id = read_identity(&image_identity)?; + references + .entry(sanitize_image_identity(&image_id)) + .or_default() + .push(sandbox_id.clone()); + let bootstrap_path = path.join(BOOTSTRAP_IDENTITY_FILE); + if regular_file_exists(&bootstrap_path)? { + let bootstrap_id = read_identity(&bootstrap_path)?; + references + .entry(sanitize_image_identity(&bootstrap_id)) + .or_default() + .push(sandbox_id); + } else { + legacy_bootstrap.push(sandbox_id); + } + } + Ok(SandboxReferences { + by_id: references, + legacy_bootstrap, + }) +} + +fn read_identity(path: &Path) -> io::Result { + if fs::metadata(path)?.len() > 1024 { + return Err(io::Error::other("sandbox image identity is too large")); + } + let value = fs::read_to_string(path)?; + let value = value.trim(); + if value.is_empty() || value.contains(['\n', '\r']) { + return Err(io::Error::other("sandbox image identity is invalid")); + } + Ok(value.to_owned()) +} + +fn directory_exists(path: &Path) -> io::Result { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => Ok(true), + Ok(_) => Err(io::Error::other(format!( + "{} is not a directory", + path.display() + ))), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +fn regular_file_exists(path: &Path) -> io::Result { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_file() => Ok(true), + Ok(_) => Err(io::Error::other(format!( + "{} is not a regular file", + path.display() + ))), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +fn require_regular_file(path: &Path) -> io::Result<()> { + if regular_file_exists(path)? { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("cache disk missing at {}", path.display()), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::symlink; + + fn list_locked(root: &Path) -> io::Result> { + let lease = lock_cache(root, LockMode::Shared)?; + let entries = lease.list()?; + drop(lease); + Ok(entries) + } + + fn prune_locked(root: &Path, dry_run: bool) -> io::Result { + let mode = if dry_run { + LockMode::Shared + } else { + LockMode::Exclusive + }; + lock_cache(root, mode)?.prune_unused(dry_run) + } + + fn cache_disk(root: &Path, id: &str) { + let dir = image_cache_root_dir(root).join(id); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join(IMAGE_CACHE_ROOTFS_IMAGE), b"cached disk").unwrap(); + } + + fn sandbox(root: &Path, id: &str, image_id: Option<&str>, bootstrap_id: Option<&str>) { + let dir = sandboxes_root_dir(root).join(id); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join(SANDBOX_REQUEST_FILE), b"request").unwrap(); + if let Some(image_id) = image_id { + fs::write(dir.join(IMAGE_IDENTITY_FILE), image_id).unwrap(); + } + if let Some(bootstrap_id) = bootstrap_id { + fs::write(dir.join(BOOTSTRAP_IDENTITY_FILE), bootstrap_id).unwrap(); + } + } + + #[test] + fn list_and_prune_preserve_kept_sandboxes() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let used = "sandbox-prepared-rootfs-ext4-umoci-v4-openshell-test-used"; + let unused = "sandbox-prepared-rootfs-ext4-umoci-v3-openshell-test-unused"; + let bootstrap = "sandbox-bootstrap-rootfs-ext4-v6-openshell-test-boot"; + for id in [used, unused, bootstrap] { + cache_disk(root, id); + } + sandbox(root, "kept-stopped", Some(used), Some(bootstrap)); + record_use(root, unused, "example/agent:v1").unwrap(); + let entries = list_locked(root).unwrap(); + assert_eq!(entries.len(), 3); + let unused_entry = entries.iter().find(|entry| entry.id == unused).unwrap(); + assert_eq!(unused_entry.image_ref.as_deref(), Some("example/agent:v1")); + assert!(unused_entry.last_used_unix_ms.is_some()); + assert!(unused_entry.allocated_bytes > 0); + assert_eq!( + unused_entry.cache_version, + "sandbox-prepared-rootfs-ext4-umoci-v3" + ); + assert!(unused_entry.sandboxes.is_empty()); + assert_eq!( + entries + .iter() + .find(|entry| entry.id == used) + .unwrap() + .sandboxes, + ["kept-stopped"] + ); + assert_eq!( + entries + .iter() + .find(|entry| entry.id == bootstrap) + .unwrap() + .sandboxes, + ["kept-stopped"] + ); + + let dry_run = prune_locked(root, true).unwrap(); + assert_eq!(dry_run.entries.len(), 1); + assert!(image_cache_dir(root, unused).exists()); + let pruned = prune_locked(root, false).unwrap(); + assert_eq!(pruned.entries.len(), 1); + assert!(!image_cache_dir(root, unused).exists()); + assert!(image_cache_dir(root, used).exists()); + assert!(image_cache_dir(root, bootstrap).exists()); + } + + #[test] + fn legacy_sandbox_protects_all_bootstrap_disks() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let bootstrap = "sandbox-bootstrap-rootfs-ext4-v5-openshell-old"; + cache_disk(root, bootstrap); + sandbox(root, "legacy-stopped", Some("sandbox-prepared-old"), None); + assert_eq!(list_locked(root).unwrap()[0].sandboxes, ["legacy-stopped"]); + assert!(prune_locked(root, false).unwrap().entries.is_empty()); + } + + #[test] + fn incomplete_sandbox_blocks_prune() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let unused = "sandbox-prepared-rootfs-ext4-v4-openshell-unused"; + cache_disk(root, unused); + sandbox(root, "incomplete", None, None); + assert!(prune_locked(root, false).is_err()); + assert!(image_cache_dir(root, unused).exists()); + } + + #[test] + fn symlinked_cache_disk_blocks_prune() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let id = "sandbox-prepared-rootfs-ext4-v4-openshell-link"; + let dir = image_cache_dir(root, id); + fs::create_dir_all(&dir).unwrap(); + let outside = root.join("outside"); + fs::write(&outside, b"keep").unwrap(); + symlink(&outside, dir.join(IMAGE_CACHE_ROOTFS_IMAGE)).unwrap(); + assert!(prune_locked(root, false).is_err()); + assert_eq!(fs::read(outside).unwrap(), b"keep"); + } + + #[test] + fn live_driver_lease_blocks_prune_lease() { + let temp = tempfile::tempdir().unwrap(); + let driver = lock_cache(temp.path(), LockMode::Shared).unwrap(); + let _reader = lock_cache(temp.path(), LockMode::Shared).unwrap(); + assert_eq!( + driver.prune_unused(false).err().unwrap().kind(), + io::ErrorKind::WouldBlock + ); + assert_eq!( + lock_cache(temp.path(), LockMode::Exclusive) + .err() + .unwrap() + .kind(), + io::ErrorKind::WouldBlock + ); + } +} diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 0ab4743771..7e3b060ae8 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -1,11 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use clap::Parser; +use clap::{Parser, Subcommand}; use futures::Stream; use miette::{IntoDiagnostic, Result}; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_vm::driver::image_cache_admin::{self, LockMode}; #[cfg(target_os = "macos")] use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; use openshell_driver_vm::{ @@ -25,6 +26,9 @@ use tracing::info; #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { + #[command(subcommand)] + cache_command: Option, + /// Operator-owned JSON policy; omitted means driver config disabled and labels required. #[arg( long, @@ -287,9 +291,90 @@ struct Args { vm_mount: Vec, } +#[derive(Debug, Subcommand)] +enum CacheCommand { + /// Inspect or prune the VM driver's prepared-image cache. + ImageCache { + #[command(subcommand)] + action: ImageCacheAction, + }, +} + +#[derive(Debug, Subcommand)] +enum ImageCacheAction { + /// List committed root disks and their persisted sandbox references. + List, + /// Remove only entries unreferenced by persisted sandboxes. + Prune { + #[arg(long, required = true)] + unused: bool, + #[arg(long)] + dry_run: bool, + }, +} + +fn run_cache_command(state_dir: &Path, command: &CacheCommand) -> Result<()> { + match command { + CacheCommand::ImageCache { + action: ImageCacheAction::List, + } => { + let lease = + image_cache_admin::lock_cache(state_dir, LockMode::Shared).into_diagnostic()?; + println!( + "ENTRY\tIMAGE\tALLOCATED_BYTES\tAPPARENT_BYTES\tLAST_USED_UNIX_MS\tCACHE_VERSION\tSANDBOXES" + ); + for entry in lease.list().into_diagnostic()? { + let image = entry.image_ref.unwrap_or_else(|| "-".to_owned()); + let image = image.replace(['\t', '\n', '\r'], " "); + let sandboxes = if entry.sandboxes.is_empty() { + "-".to_owned() + } else { + entry.sandboxes.join(",") + }; + println!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}", + entry.id, + image, + entry.allocated_bytes, + entry.apparent_bytes, + entry + .last_used_unix_ms + .map_or_else(|| "-".to_owned(), |value| value.to_string()), + entry.cache_version, + sandboxes, + ); + } + } + CacheCommand::ImageCache { + action: ImageCacheAction::Prune { unused, dry_run }, + } => { + if !unused { + return Err(miette::miette!("prune requires --unused")); + } + let mode = if *dry_run { + LockMode::Shared + } else { + LockMode::Exclusive + }; + let lease = image_cache_admin::lock_cache(state_dir, mode).into_diagnostic()?; + let report = lease.prune_unused(*dry_run).into_diagnostic()?; + let action = if *dry_run { "would remove" } else { "removed" }; + println!( + "{action} {} entries, {} allocated bytes", + report.entries.len(), + report.allocated_bytes, + ); + } + } + Ok(()) +} + #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); + if let Some(command) = &args.cache_command { + return run_cache_command(&args.state_dir, command); + } if let Some(request) = args.internal_prepare_image { openshell_driver_vm::driver::run_image_preparation_worker(&request) .await @@ -305,6 +390,9 @@ async fn main() -> Result<()> { return Ok(()); } + let _cache_lease = + image_cache_admin::lock_cache(&args.state_dir, LockMode::Shared).into_diagnostic()?; + let _tracing = openshell_otel::install_driver_tracing( openshell_driver_vm::otel_tracing::TRACING, openshell_otel::DriverTracingConfig { @@ -799,12 +887,50 @@ fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { #[cfg(test)] mod tests { use super::{ - Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials, - compute_driver_listen_mode, parse_vm_mount_arg, + Args, CacheCommand, ComputeDriverListenMode, ImageCacheAction, PeerCredentials, + authorize_peer_credentials, compute_driver_listen_mode, parse_vm_mount_arg, }; use clap::Parser; use std::path::PathBuf; + #[test] + fn image_cache_commands_parse_and_prune_requires_unused() { + let list = Args::try_parse_from([ + "openshell-driver-vm", + "--state-dir", + "/tmp/vm-state", + "image-cache", + "list", + ]) + .unwrap(); + assert!(matches!( + list.cache_command, + Some(CacheCommand::ImageCache { + action: ImageCacheAction::List + }) + )); + assert_eq!(list.state_dir, PathBuf::from("/tmp/vm-state")); + + assert!(Args::try_parse_from(["openshell-driver-vm", "image-cache", "prune"]).is_err()); + let prune = Args::try_parse_from([ + "openshell-driver-vm", + "image-cache", + "prune", + "--unused", + "--dry-run", + ]) + .unwrap(); + assert!(matches!( + prune.cache_command, + Some(CacheCommand::ImageCache { + action: ImageCacheAction::Prune { + unused: true, + dry_run: true + } + }) + )); + } + #[test] fn corporate_proxy_flags_parse_into_driver_settings() { let args = Args::parse_from([ diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 70f6891ce1..b91e100267 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -234,6 +234,20 @@ The driver looks up sandbox images in local Docker or Podman before pulling from Registry downloads must match the resolved image digest, including the selected platform for multi-platform images. A mismatch fails image preparation. After upgrading, the driver rebuilds older bootstrap and prepared-image cache entries on first use, so the first sandbox start can take longer. +The VM driver keeps old prepared disks until you prune them. On the gateway +host, use the VM driver binary with the gateway's configured `state_dir` to +inspect the cache and preview unused entries: + +```shell +openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache list +openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache prune --unused --dry-run +``` + +Stop the VM gateway before running `image-cache prune --unused` without +`--dry-run`. Prune preserves disks referenced by kept sandboxes, including +stopped ones. Older entries can lack an image reference or last-use timestamp +in the listing. + VM sandboxes have no network interface. All traffic flows through the OpenShell supervisor on the host. For networks that require a corporate proxy, the VM driver accepts the same `https_proxy` and `proxy_*` options as Podman. To reach a proxy on the gateway host, use `http://host.openshell.internal:`. If the gateway is briefly unavailable while a VM starts, its supervisor retries the session connection. The sandbox stays in provisioning until the gateway accepts the session. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 9e56f7117c..cf22501f21 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1494,6 +1494,13 @@ Use the VM driver logs and host diagnostics available in the user's environment. - Host virtualization support is enabled. - The sandbox supervisor can establish its authenticated gateway session. +If the VM host reports `No space left on device`, inspect committed image disks +with `openshell-driver-vm --state-dir image-cache list`. +Preview cleanup with `image-cache prune --unused --dry-run` using the same +`--state-dir`. Stop the VM gateway before running `image-cache prune --unused` +without `--dry-run`; the driver protects persisted sandbox references and +refuses cleanup while its process holds the cache lease. + For `vm.mounts` (virtiofs host bind mounts) failures, also verify: - `[openshell.drivers.vm]` sets `allow_driver_config = true` and `enable_bind_mounts = true`; an external VM driver needs `--enable-bind-mounts` and `allow_driver_config` in `--admission-config-json`. From d7e3a24ab47729b4335253522a9320286e2a8c0e Mon Sep 17 00:00:00 2001 From: Vineeth Sai Narajala Date: Mon, 2 Nov 2026 22:08:00 +0000 Subject: [PATCH 2/2] fix(vm): protect cache references and validate cache state Signed-off-by: Vineeth Sai Narajala --- crates/openshell-driver-vm/README.md | 21 ++-- crates/openshell-driver-vm/src/driver.rs | 105 +++++++++++++++--- .../src/image_cache_admin.rs | 43 ++++++- crates/openshell-driver-vm/src/main.rs | 30 ++--- .../tests/image_cache_cli.rs | 46 ++++++++ docs/how-it-works/sandboxes/runtimes.mdx | 14 ++- skills/debug-openshell-cluster/SKILL.md | 3 +- 7 files changed, 212 insertions(+), 50 deletions(-) create mode 100644 crates/openshell-driver-vm/tests/image_cache_cli.rs diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index a379e9f819..e47df94515 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -166,7 +166,7 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`, | Configuration key | Default | Purpose | |---|---|---| | `grpc_endpoint` | empty | Required. URL the native host supervisor uses to reach the gateway. Host loopback such as `http://127.0.0.1:` is valid. Legacy guest aliases are normalized to host loopback. This endpoint is never sent into the VM. | -| `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. Relative paths are resolved to absolute paths at driver startup. | +| `state_dir` | OpenShell state directory under `vm-driver` (fallback: `target/openshell-vm-driver`) | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. Relative paths are resolved to absolute paths at driver startup. | | `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. | | `default_image` | OpenShell base image | Sandbox image used when a create request omits one. | | `bootstrap_image` | unset | VM runtime image used as the immutable bootstrap root disk. Falls back only to the operator-configured `default_image`; the driver refuses to start when both are empty. | @@ -241,8 +241,8 @@ At startup, the driver reclaims inactive attempts in this directory. It preserve ### Inspect and prune image disks -Run the VM driver binary on the gateway host with the same `state_dir` configured -for the gateway: +Run the VM driver binary on the gateway host. Pass the gateway's existing +`state_dir` explicitly; cache commands reject an omitted or missing state directory: ```shell openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache list @@ -252,8 +252,10 @@ openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache prune --unu `list` reports each committed root disk's cache ID, image reference, allocated and apparent bytes, last-use Unix timestamp in milliseconds, cache layout version, and persisted sandbox IDs that reference it. Older entries without -cache metadata show `-` for the image reference and last use. New cache uses -record this metadata without changing the disk contents. +cache metadata show `-` for the image reference and last use. Disks prepared +from a rootfs tar show `-` for image reference because no container image +produced them. New cache uses record this metadata without changing the disk +contents. Stop the VM gateway before removing entries, then run: @@ -265,10 +267,11 @@ Prune removes only committed disks that no persisted sandbox references. It leaves disks referenced by stopped sandboxes in place. For sandboxes created before bootstrap identities were persisted, it conservatively preserves all bootstrap disks. Prune refuses to run while a current driver holds the state -lock or an older driver is reachable at the default state socket. The reported -bytes are the disks' allocated bytes before removal; filesystem snapshots and -shared blocks can make actual free space differ. Nothing is removed -automatically. +lock or an older driver is reachable at the default state socket. Stop older +standalone drivers that use a custom socket explicitly; their socket is outside +this check. The reported bytes are the disks' allocated bytes before removal. +Filesystem snapshots and shared blocks can make actual free space differ. +Nothing is removed automatically. Each sandbox gets its own sparse writable `/sandboxes//overlay.ext4`. Guest init mounts overlayfs as `/` diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 399e46af00..4353c88079 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1876,13 +1876,17 @@ impl VmDriver { if let Err(error) = image_cache_admin::record_use( &self.config.state_dir, &image_plan.bootstrap_image_identity, - &bootstrap_image_ref, + Some(bootstrap_image_ref.as_str()), ) { warn!(%error, "vm driver: could not record bootstrap cache use"); } if image_plan.image_disk.is_some() - && let Err(error) = - image_cache_admin::record_use(&self.config.state_dir, &image_identity, &image_ref) + && let Err(error) = record_prepared_image_use( + &self.config.state_dir, + &image_identity, + &image_ref, + driver_config_had_rootfs_tar, + ) { warn!(%error, "vm driver: could not record prepared image cache use"); } @@ -7360,31 +7364,57 @@ fn unique_image_cache_suffix() -> String { format!("{}-{counter}", openshell_core::time::now_ms()) } +fn record_prepared_image_use( + state_dir: &Path, + image_identity: &str, + image_ref: &str, + from_rootfs_tar: bool, +) -> Result<(), std::io::Error> { + image_cache_admin::record_use( + state_dir, + image_identity, + (!from_rootfs_tar).then_some(image_ref), + ) +} + +async fn write_sandbox_identity_file( + state_dir: &Path, + filename: &str, + identity: &str, +) -> Result<(), std::io::Error> { + let sequence = IMAGE_CACHE_BUILD_COUNTER.fetch_add(1, Ordering::Relaxed); + let temporary = state_dir.join(format!(".{filename}.{}.{sequence}.tmp", std::process::id())); + let result = async { + write_private_file(&temporary, format!("{identity}\n").into_bytes()).await?; + tokio::fs::rename(&temporary, state_dir.join(filename)).await + } + .await; + if result.is_err() { + let _ = tokio::fs::remove_file(&temporary).await; + } + result +} + async fn write_sandbox_image_metadata( state_dir: &Path, image_ref: &str, image_identity: &str, bootstrap_image_identity: &str, ) -> Result<(), std::io::Error> { - tokio::fs::write( - state_dir.join(IMAGE_IDENTITY_FILE), - format!("{image_identity}\n"), + // Write the bootstrap reference first. If this fails, the old image + // identity still protects its disk when prune scans persisted state. + write_sandbox_identity_file( + state_dir, + image_cache_admin::BOOTSTRAP_IDENTITY_FILE, + bootstrap_image_identity, ) .await?; + write_sandbox_identity_file(state_dir, IMAGE_IDENTITY_FILE, image_identity).await?; tokio::fs::write( state_dir.join(IMAGE_REFERENCE_FILE), format!("{image_ref}\n"), ) .await?; - if let Err(error) = tokio::fs::write( - state_dir.join(image_cache_admin::BOOTSTRAP_IDENTITY_FILE), - format!("{bootstrap_image_identity}\n"), - ) - .await - { - warn!(%error, "vm driver: could not persist bootstrap cache identity"); - } - Ok(()) } @@ -9152,6 +9182,51 @@ mod tests { } } + #[tokio::test] + async fn failed_bootstrap_identity_write_preserves_prior_image_identity() { + let temp = tempfile::tempdir().unwrap(); + let state_dir = temp.path(); + tokio::fs::write(state_dir.join(IMAGE_IDENTITY_FILE), b"prior-image\n") + .await + .unwrap(); + tokio::fs::create_dir(state_dir.join(image_cache_admin::BOOTSTRAP_IDENTITY_FILE)) + .await + .unwrap(); + + assert!( + write_sandbox_image_metadata( + state_dir, + "new-image", + "new-image-id", + "new-bootstrap-id", + ) + .await + .is_err() + ); + assert_eq!( + tokio::fs::read_to_string(state_dir.join(IMAGE_IDENTITY_FILE)) + .await + .unwrap(), + "prior-image\n" + ); + } + + #[test] + fn rootfs_tar_cache_use_has_no_image_reference() { + let temp = tempfile::tempdir().unwrap(); + let cache_id = "sandbox-prepared-rootfs-ext4-umoci-v4-openshell-rootfs-tar"; + let disk = image_cache_rootfs_image(temp.path(), cache_id); + fs::create_dir_all(disk.parent().unwrap()).unwrap(); + fs::write(&disk, b"disk").unwrap(); + + record_prepared_image_use(temp.path(), cache_id, "unrelated/image:tag", true).unwrap(); + let lease = image_cache_admin::lock_cache(temp.path(), image_cache_admin::LockMode::Shared) + .unwrap(); + let entry = lease.list().unwrap().pop().unwrap(); + assert_eq!(entry.image_ref, None); + assert!(entry.last_used_unix_ms.is_some()); + } + async fn assert_inactive_restore_retains_identity(marker: &str, reason: &str) { let temp = tempfile::tempdir().unwrap(); let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); diff --git a/crates/openshell-driver-vm/src/image_cache_admin.rs b/crates/openshell-driver-vm/src/image_cache_admin.rs index ab1e6688de..2fe0e140ab 100644 --- a/crates/openshell-driver-vm/src/image_cache_admin.rs +++ b/crates/openshell-driver-vm/src/image_cache_admin.rs @@ -60,8 +60,23 @@ impl CacheLease { /// A shared lease covers the driver and its image workers. Prune takes the /// exclusive lease, so it cannot remove a disk being prepared or attached. pub fn lock_cache(state_dir: &Path, mode: LockMode) -> io::Result { - fs::create_dir_all(state_dir)?; + lock_cache_inner(state_dir, mode, true) +} + +/// Cache commands use an existing state directory, so a typo cannot create a +/// new empty cache and report a misleading successful inspection. +pub fn lock_existing_cache(state_dir: &Path, mode: LockMode) -> io::Result { + lock_cache_inner(state_dir, mode, false) +} + +fn lock_cache_inner(state_dir: &Path, mode: LockMode, create: bool) -> io::Result { + if create { + fs::create_dir_all(state_dir)?; + } let state_dir = fs::canonicalize(state_dir)?; + if !fs::metadata(&state_dir)?.is_dir() { + return Err(io::Error::other("VM driver state path is not a directory")); + } let file = OpenOptions::new() .read(true) .write(true) @@ -126,7 +141,7 @@ fn refuse_live_legacy_driver(state_dir: &Path) -> io::Result<()> { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct CacheMetadata { - image_ref: String, + image_ref: Option, last_used_unix_ms: u64, } @@ -149,14 +164,14 @@ pub struct PruneReport { /// Metadata is best effort: a full disk must not prevent an otherwise ready /// sandbox from starting. It is never used to decide whether pruning is safe. -pub fn record_use(state_dir: &Path, id: &str, image_ref: &str) -> io::Result<()> { +pub fn record_use(state_dir: &Path, id: &str, image_ref: Option<&str>) -> io::Result<()> { let dir = image_cache_dir(state_dir, id); require_regular_file(&dir.join(IMAGE_CACHE_ROOTFS_IMAGE))?; let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .map_err(io::Error::other)?; let metadata = CacheMetadata { - image_ref: image_ref.to_owned(), + image_ref: image_ref.map(str::to_owned), last_used_unix_ms: u64::try_from(timestamp.as_millis()).map_err(io::Error::other)?, }; let sequence = METADATA_WRITE_COUNTER.fetch_add(1, Ordering::Relaxed); @@ -223,7 +238,7 @@ fn list(state_dir: &Path) -> io::Result> { .split_once("-openshell-") .map_or_else(|| "unknown".to_owned(), |(version, _)| version.to_owned()), id, - image_ref: metadata.as_ref().map(|value| value.image_ref.clone()), + image_ref: metadata.as_ref().and_then(|value| value.image_ref.clone()), allocated_bytes: disk_metadata.blocks().saturating_mul(512), apparent_bytes: disk_metadata.len(), last_used_unix_ms: metadata.map(|value| value.last_used_unix_ms), @@ -418,7 +433,7 @@ mod tests { cache_disk(root, id); } sandbox(root, "kept-stopped", Some(used), Some(bootstrap)); - record_use(root, unused, "example/agent:v1").unwrap(); + record_use(root, unused, Some("example/agent:v1")).unwrap(); let entries = list_locked(root).unwrap(); assert_eq!(entries.len(), 3); let unused_entry = entries.iter().find(|entry| entry.id == unused).unwrap(); @@ -493,6 +508,22 @@ mod tests { assert_eq!(fs::read(outside).unwrap(), b"keep"); } + #[test] + fn live_legacy_driver_socket_blocks_prune() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let unused = "sandbox-prepared-rootfs-ext4-v4-openshell-unused"; + cache_disk(root, unused); + let run_dir = root.join("run"); + fs::create_dir(&run_dir).unwrap(); + let _listener = + std::os::unix::net::UnixListener::bind(run_dir.join("compute-driver.sock")).unwrap(); + + let error = prune_locked(root, false).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::WouldBlock); + assert!(image_cache_dir(root, unused).exists()); + } + #[test] fn live_driver_lease_blocks_prune_lease() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 18edcdd9f1..251bda4306 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -117,12 +117,8 @@ struct Args { #[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")] bootstrap_image: String, - #[arg( - long, - env = "OPENSHELL_VM_DRIVER_STATE_DIR", - default_value = "target/openshell-vm-driver" - )] - state_dir: PathBuf, + #[arg(long, env = "OPENSHELL_VM_DRIVER_STATE_DIR")] + state_dir: Option, #[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")] guest_tls_ca: Option, @@ -306,8 +302,8 @@ fn run_cache_command(state_dir: &Path, command: &CacheCommand) -> Result<()> { CacheCommand::ImageCache { action: ImageCacheAction::List, } => { - let lease = - image_cache_admin::lock_cache(state_dir, LockMode::Shared).into_diagnostic()?; + let lease = image_cache_admin::lock_existing_cache(state_dir, LockMode::Shared) + .into_diagnostic()?; println!( "ENTRY\tIMAGE\tALLOCATED_BYTES\tAPPARENT_BYTES\tLAST_USED_UNIX_MS\tCACHE_VERSION\tSANDBOXES" ); @@ -344,7 +340,8 @@ fn run_cache_command(state_dir: &Path, command: &CacheCommand) -> Result<()> { } else { LockMode::Exclusive }; - let lease = image_cache_admin::lock_cache(state_dir, mode).into_diagnostic()?; + let lease = + image_cache_admin::lock_existing_cache(state_dir, mode).into_diagnostic()?; let report = lease.prune_unused(*dry_run).into_diagnostic()?; let action = if *dry_run { "would remove" } else { "removed" }; println!( @@ -361,8 +358,15 @@ fn run_cache_command(state_dir: &Path, command: &CacheCommand) -> Result<()> { async fn main() -> Result<()> { let args = Args::parse(); if let Some(command) = &args.cache_command { - return run_cache_command(&args.state_dir, command); + let state_dir = args.state_dir.as_deref().ok_or_else(|| { + miette::miette!("image-cache requires --state-dir or OPENSHELL_VM_DRIVER_STATE_DIR") + })?; + return run_cache_command(state_dir, command); } + let state_dir = args + .state_dir + .clone() + .unwrap_or_else(|| PathBuf::from("target/openshell-vm-driver")); if let Some(request) = args.internal_prepare_image { openshell_driver_vm::driver::run_image_preparation_worker(&request) .await @@ -379,7 +383,7 @@ async fn main() -> Result<()> { } let _cache_lease = - image_cache_admin::lock_cache(&args.state_dir, LockMode::Shared).into_diagnostic()?; + image_cache_admin::lock_cache(&state_dir, LockMode::Shared).into_diagnostic()?; let _tracing = openshell_otel::install_driver_tracing( openshell_driver_vm::otel_tracing::TRACING, @@ -416,7 +420,7 @@ async fn main() -> Result<()> { .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, supervisor_otlp, supervisor_gateway_name: args.gateway_name.clone(), - state_dir: args.state_dir.clone(), + state_dir, launcher_bin: None, default_image: args.default_image.clone(), bootstrap_image: args.bootstrap_image.clone(), @@ -890,7 +894,7 @@ mod tests { action: ImageCacheAction::List }) )); - assert_eq!(list.state_dir, PathBuf::from("/tmp/vm-state")); + assert_eq!(list.state_dir, Some(PathBuf::from("/tmp/vm-state"))); assert!(Args::try_parse_from(["openshell-driver-vm", "image-cache", "prune"]).is_err()); let prune = Args::try_parse_from([ diff --git a/crates/openshell-driver-vm/tests/image_cache_cli.rs b/crates/openshell-driver-vm/tests/image_cache_cli.rs new file mode 100644 index 0000000000..bb3ffd4ddc --- /dev/null +++ b/crates/openshell-driver-vm/tests/image_cache_cli.rs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::process::Command; + +#[test] +fn cache_commands_require_an_existing_explicit_state_dir() { + let temp = tempfile::tempdir().unwrap(); + let missing = temp.path().join("missing-state"); + let binary = env!("CARGO_BIN_EXE_openshell-driver-vm"); + + let without_state_dir = Command::new(binary) + .env_remove("OPENSHELL_VM_DRIVER_STATE_DIR") + .current_dir(temp.path()) + .args(["image-cache", "list"]) + .output() + .unwrap(); + assert!(!without_state_dir.status.success()); + assert!(!temp.path().join("target/openshell-vm-driver").exists()); + + let missing_state_dir = Command::new(binary) + .env_remove("OPENSHELL_VM_DRIVER_STATE_DIR") + .arg("--state-dir") + .arg(&missing) + .args(["image-cache", "prune", "--unused", "--dry-run"]) + .output() + .unwrap(); + assert!(!missing_state_dir.status.success()); + assert!(!missing.exists()); + + let existing = temp.path().join("existing-state"); + std::fs::create_dir(&existing).unwrap(); + let valid_list = Command::new(binary) + .env_remove("OPENSHELL_VM_DRIVER_STATE_DIR") + .arg("--state-dir") + .arg(&existing) + .args(["image-cache", "list"]) + .output() + .unwrap(); + assert!( + valid_list.status.success(), + "{}", + String::from_utf8_lossy(&valid_list.stderr) + ); + assert!(String::from_utf8_lossy(&valid_list.stdout).starts_with("ENTRY IMAGE ")); +} diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index b91e100267..284fa4b635 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -234,19 +234,21 @@ The driver looks up sandbox images in local Docker or Podman before pulling from Registry downloads must match the resolved image digest, including the selected platform for multi-platform images. A mismatch fails image preparation. After upgrading, the driver rebuilds older bootstrap and prepared-image cache entries on first use, so the first sandbox start can take longer. -The VM driver keeps old prepared disks until you prune them. On the gateway -host, use the VM driver binary with the gateway's configured `state_dir` to -inspect the cache and preview unused entries: +The VM driver keeps old image disks until you prune them. On the gateway +host, pass the gateway's existing `state_dir` explicitly to the VM driver +binary to inspect the cache and preview unused entries: ```shell openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache list openshell-driver-vm --state-dir /path/to/vm-driver-state image-cache prune --unused --dry-run ``` -Stop the VM gateway before running `image-cache prune --unused` without -`--dry-run`. Prune preserves disks referenced by kept sandboxes, including +Stop the VM gateway and any older standalone VM driver using the same +state directory before running `image-cache prune --unused` without `--dry-run`. +Prune preserves disks referenced by kept sandboxes, including stopped ones. Older entries can lack an image reference or last-use timestamp -in the listing. +in the listing. Disks prepared from a rootfs tar have no container image +reference in the listing. VM sandboxes have no network interface. All traffic flows through the OpenShell supervisor on the host. For networks that require a corporate proxy, the VM driver accepts the same `https_proxy` and `proxy_*` options as Podman. To reach a proxy on the gateway host, use `http://host.openshell.internal:`. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 3335abf8cd..265d6e8f69 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1497,7 +1497,8 @@ Use the VM driver logs and host diagnostics available in the user's environment. If the VM host reports `No space left on device`, inspect committed image disks with `openshell-driver-vm --state-dir image-cache list`. Preview cleanup with `image-cache prune --unused --dry-run` using the same -`--state-dir`. Stop the VM gateway before running `image-cache prune --unused` +`--state-dir`. The state directory must already exist. Stop the VM gateway and +any older standalone VM driver before running `image-cache prune --unused` without `--dry-run`; the driver protects persisted sandbox references and refuses cleanup while its process holds the cache lease.