libvirt: Retry virsh when the session daemon is restarting

CI intermittently fails bcvk libvirt commands with "Failed to connect
socket to '/run/user/1001/libvirt/libvirt-sock': No such file or
directory" (bootc-dev/bootc#1843), typically right after a step that
used no libvirt for two minutes, such as creating a base disk.

The libvirt client auto-spawns the qemu:///session daemon with
--timeout=120. When that daemon exits on idle it removes its socket
first and releases its pidfile only after its drivers shut down. A
client connecting in that window spawns a new daemon, which exits at
once because the pidfile is still held, then waits five seconds for a
socket nobody will create.

A connection failure means virsh never ran its command, so it is safe
to run it again; the retry spawns the daemon anew once the old one is
gone. Route every virsh invocation through one VirshCommand type that
does that for session connections, with a bounded backoff. It doesn't
deref to Command, so a call chain can't bypass the retry.

Generated-by: AI
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters-bot

crates/kit/src/domain_list.rs

@@ -3,11 +3,11 @@
 //! This module provides functionality to list libvirt domains created by bcvk libvirt,
 //! using libvirt as the source of truth instead of the VmRegistry cache.
 
+use crate::libvirt::virsh::VirshCommand;
 use crate::xml_utils;
 use base64::Engine;
 use color_eyre::{eyre::Context, Result};
 use serde::{Deserialize, Serialize};
-use std::process::Command;
 use std::time::SystemTime;
 
 /// Information about a podman-bootc domain from libvirt
@@ -87,13 +87,8 @@ impl DomainLister {
     }
 
     /// Build a virsh command with optional connection URI
-    fn virsh_command(&self) -> Command {
-        let mut cmd = Command::new("virsh");
-        cmd.env("LC_ALL", "C");
-        if let Some(ref uri) = self.connect_uri {
-            cmd.arg("-c").arg(uri);
-        }
-        cmd
+    fn virsh_command(&self) -> VirshCommand {
+        VirshCommand::new(self.connect_uri.as_deref())
     }
 
     /// List all domains (running and inactive)

crates/kit/src/libvirt/list_volumes.rs

@@ -3,11 +3,11 @@
 //! This module provides functionality to discover and list bootc volumes
 //! with their container image metadata and creation information.
 
+use crate::libvirt::virsh::VirshCommand;
 use clap::Parser;
 use color_eyre::{eyre::eyre, Result};
 use comfy_table::{presets::UTF8_FULL, Table};
 use serde_json::{json, Value};
-use std::process::Command;
 use tracing::{debug, warn};
 
 /// Configuration options for listing bootc volumes
@@ -71,7 +71,7 @@ impl BootcVolume {
 
 impl LibvirtListVolumesOpts {
     /// Build a virsh command with optional connection URI
-    fn virsh_command(&self, global_opts: &crate::libvirt::LibvirtOptions) -> Command {
+    fn virsh_command(&self, global_opts: &crate::libvirt::LibvirtOptions) -> VirshCommand {
         global_opts.virsh_command()
     }
 

crates/kit/src/libvirt/mod.rs

@@ -40,6 +40,7 @@ pub mod start;
 pub mod status;
 pub mod stop;
 pub mod upload;
+pub mod virsh;
 
 /// Global options for libvirt operations
 #[derive(Debug, Clone, Default)]
@@ -50,13 +51,8 @@ pub struct LibvirtOptions {
 
 impl LibvirtOptions {
     /// Create a virsh Command with the appropriate connection URI
-    pub fn virsh_command(&self) -> std::process::Command {
-        let mut cmd = std::process::Command::new("virsh");
-        cmd.env("LC_ALL", "C");
-        if let Some(ref uri) = self.connect {
-            cmd.arg("-c").arg(uri);
-        }
-        cmd
+    pub fn virsh_command(&self) -> virsh::VirshCommand {
+        virsh::VirshCommand::new(self.connect.as_deref())
     }
 }
 

crates/kit/src/libvirt/run.rs

@@ -27,13 +27,8 @@ const SSH_WAIT_TIMEOUT_SECONDS: u64 = 180;
 const UPDATE_FROM_HOST_TRANSPORT: &str = "containers-storage";
 
 /// Create a virsh command with optional connection URI
-pub(super) fn virsh_command(connect_uri: Option<&str>) -> Result<std::process::Command> {
-    let mut cmd = std::process::Command::new("virsh");
-    cmd.env("LC_ALL", "C");
-    if let Some(uri) = connect_uri {
-        cmd.arg("-c").arg(uri);
-    }
-    Ok(cmd)
+pub(super) fn virsh_command(connect_uri: Option<&str>) -> Result<super::virsh::VirshCommand> {
+    Ok(super::virsh::VirshCommand::new(connect_uri))
 }
 
 /// Run a virsh command and handle errors consistently

crates/kit/src/libvirt/status.rs

@@ -3,10 +3,10 @@
 //! This module provides a status command that outputs JSON metadata about
 //! the libvirt environment, including version information and domain count.
 
+use crate::libvirt::virsh::VirshCommand;
 use clap::Parser;
 use color_eyre::{eyre::Context, Result};
 use serde::{Deserialize, Serialize};
-use std::process::Command;
 
 use crate::domain_list::DomainLister;
 
@@ -98,7 +98,7 @@ fn parse_libvirt_version_from_output(version_output: &str) -> Option<LibvirtVers
 
 /// Parse libvirt version from virsh version output
 pub fn parse_libvirt_version() -> Result<Option<LibvirtVersion>> {
-    let output = Command::new("virsh")
+    let output = VirshCommand::new(None)
         .args(&["version"])
         .output()
         .with_context(|| "Failed to check libvirt version")?;

crates/kit/src/libvirt/upload.rs

@@ -5,13 +5,13 @@
 
 use crate::common_opts::MemoryOpts;
 use crate::install_options::InstallOptions;
+use crate::libvirt::virsh::VirshCommand;
 use crate::to_disk::{run as to_disk, ToDiskAdditionalOpts, ToDiskOpts};
 use crate::{images, utils};
 use camino::Utf8PathBuf;
 use clap::Parser;
 use color_eyre::{eyre::eyre, Result};
 use std::path::Path;
-use std::process::Command;
 use tracing::debug;
 
 /// Configuration options for uploading a bootc disk image to libvirt
@@ -46,7 +46,7 @@ pub struct LibvirtUploadOpts {
 
 impl LibvirtUploadOpts {
     /// Build a virsh command with optional connection URI  
-    fn virsh_command(&self, global_opts: &crate::libvirt::LibvirtOptions) -> Command {
+    fn virsh_command(&self, global_opts: &crate::libvirt::LibvirtOptions) -> VirshCommand {
         global_opts.virsh_command()
     }