Skip to content

Commit 9291888

Browse files
committed
feat(vm): adopt shared supervisor boundary mode
Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent db42b60 commit 9291888

12 files changed

Lines changed: 1009 additions & 685 deletions

File tree

‎crates/openshell-driver-vm/Cargo.toml‎

Lines changed: 66 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -17,44 +17,81 @@ path = "src/lib.rs"
1717
[[bin]]
1818
name = "openshell-driver-vm"
1919
path = "src/main.rs"
20+
required-features = ["compute-driver"]
2021

2122
[dependencies]
2223
openshell-core = { path = "../openshell-core", default-features = false }
23-
openshell-otel = { path = "../openshell-otel" }
24-
openshell-policy = { path = "../openshell-policy" }
25-
openshell-vfio = { path = "../openshell-vfio" }
24+
openshell-isolation-interface = { path = "../openshell-isolation-interface" }
25+
openshell-otel = { path = "../openshell-otel", optional = true }
26+
openshell-policy = { path = "../openshell-policy", optional = true }
27+
openshell-vfio = { path = "../openshell-vfio", optional = true }
2628

27-
bollard = { version = "0.20", features = ["ssh"] }
29+
base64 = { workspace = true, optional = true }
30+
bollard = { version = "0.20", features = ["ssh"], optional = true }
2831
tokio = { workspace = true }
29-
tonic = { workspace = true, features = ["transport"] }
30-
tower-http = { workspace = true }
31-
http = { workspace = true }
32-
prost = { workspace = true }
33-
prost-types = { workspace = true }
34-
futures = { workspace = true }
35-
tokio-stream = { workspace = true, features = ["net"] }
36-
nix = { workspace = true }
37-
clap = { workspace = true }
38-
tracing = { workspace = true }
39-
tracing-subscriber = { workspace = true }
40-
opentelemetry = { workspace = true }
41-
opentelemetry_sdk = { workspace = true }
42-
tracing-opentelemetry = { workspace = true }
43-
miette = { workspace = true }
44-
url = { workspace = true }
32+
tonic = { workspace = true, features = ["transport"], optional = true }
33+
tower-http = { workspace = true, optional = true }
34+
http = { workspace = true, optional = true }
35+
prost = { workspace = true, optional = true }
36+
prost-types = { workspace = true, optional = true }
37+
futures = { workspace = true, optional = true }
38+
tokio-stream = { workspace = true, features = ["net"], optional = true }
39+
nix = { workspace = true, optional = true }
40+
clap = { workspace = true, optional = true }
41+
tracing = { workspace = true, optional = true }
42+
tracing-subscriber = { workspace = true, optional = true }
43+
opentelemetry = { workspace = true, optional = true }
44+
opentelemetry_sdk = { workspace = true, optional = true }
45+
tracing-opentelemetry = { workspace = true, optional = true }
46+
miette = { workspace = true, optional = true }
47+
rand = { workspace = true, optional = true }
48+
url = { workspace = true, optional = true }
4549
serde = { workspace = true }
4650
serde_json = { workspace = true }
47-
oci-client = "0.16"
51+
oci-client = { version = "0.16", optional = true }
4852
libc = "0.2"
49-
rustix = { workspace = true }
50-
libloading = "0.8"
51-
tar = "0.4"
52-
flate2 = "1"
53-
sha2 = "0.10"
54-
zstd = "0.13"
53+
rustix = { workspace = true, optional = true }
54+
libloading = { version = "0.8", optional = true }
55+
tar = { version = "0.4", optional = true }
56+
flate2 = { version = "1", optional = true }
57+
sha2 = { version = "0.10", optional = true }
58+
zstd = { version = "0.13", optional = true }
5559

5660
[features]
57-
default = ["telemetry"]
61+
default = ["compute-driver", "telemetry"]
62+
## Build the standalone compute driver and its host runtime implementation.
63+
compute-driver = [
64+
"dep:base64",
65+
"dep:bollard",
66+
"dep:clap",
67+
"dep:flate2",
68+
"dep:futures",
69+
"dep:http",
70+
"dep:libloading",
71+
"dep:miette",
72+
"dep:nix",
73+
"dep:oci-client",
74+
"dep:openshell-otel",
75+
"dep:openshell-policy",
76+
"dep:openshell-vfio",
77+
"dep:opentelemetry",
78+
"dep:opentelemetry_sdk",
79+
"dep:polling",
80+
"dep:prost",
81+
"dep:prost-types",
82+
"dep:rand",
83+
"dep:rustix",
84+
"dep:sha2",
85+
"dep:tar",
86+
"dep:tokio-stream",
87+
"dep:tonic",
88+
"dep:tower-http",
89+
"dep:tracing",
90+
"dep:tracing-opentelemetry",
91+
"dep:tracing-subscriber",
92+
"dep:url",
93+
"dep:zstd",
94+
]
5895
## Compile in telemetry support (forwards to openshell-core/telemetry). On by
5996
## default; build with `--no-default-features` for a telemetry-free VM driver
6097
## that reports telemetry disabled to the sandboxes it launches.
@@ -72,7 +109,7 @@ opentelemetry_sdk = { workspace = true, features = ["testing"] }
72109
# nix::sys::prctl::set_pdeathsig there keeps the Linux path a single
73110
# syscall with no helper thread.
74111
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly"))'.dependencies]
75-
polling = "3.11"
112+
polling = { version = "3.11", optional = true }
76113

77114
[lints]
78115
workspace = true

‎crates/openshell-driver-vm/README.md‎

Lines changed: 39 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,48 @@
11
# openshell-driver-vm
22

3-
> Status: Experimental. The VM compute driver is under active development and the interface still has VM-specific plumbing that will be generalized.
3+
> Status: Experimental. The VM compute driver is under active development.
44
5-
Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess, talks to it over a Unix domain socket with the `openshell.compute.v1.ComputeDriver` gRPC surface, and lets it manage per-sandbox microVMs. The runtime (libkrun + libkrunfw + gvproxy), guest OCI unpacker, and sandbox supervisor are embedded directly in the binary; each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir.
5+
Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess and talks to it over the `openshell.compute.v1.ComputeDriver` Unix-socket surface. `openshell-sandbox --mode=control` runs as a native host process. The same binary runs as `--mode=boundary` inside each microVM and applies guest-local isolation through the shared RFC 0012 boundary protocol over virtio-vsock.
6+
7+
The driver still embeds libkrun, libkrunfw, gvproxy, the guest OCI unpacker, the portable guest leaf, and the existing custom kernel runtime. Each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir.
68

79
## How it fits together
810

911
```mermaid
1012
flowchart LR
11-
subgraph host["Host process"]
13+
subgraph host["Host"]
1214
gateway["openshell-server<br/>(compute::vm::spawn)"]
13-
driver["openshell-driver-vm<br/>├── libkrun (VM)<br/>├── gvproxy (net)<br/>└── openshell-sandbox.zst"]
15+
driver["openshell-driver-vm<br/>libkrun + gvproxy"]
16+
supervisor["openshell-sandbox<br/>logical supervisor"]
1417
gateway <-->|"gRPC over UDS<br/>compute-driver.sock"| driver
18+
supervisor <-->|"authenticated gRPC<br/>policy + relay"| gateway
1519
end
1620
1721
subgraph guest["Per-sandbox microVM"]
1822
init["/srv/openshell-vm-<br/>sandbox-init.sh"]
19-
supervisor["/opt/openshell/bin/<br/>openshell-sandbox<br/>(PID 1)"]
20-
init --> supervisor
23+
leaf["openshell-sandbox --mode=boundary<br/>boundary role (PID 1)"]
24+
workload["sandbox workload"]
25+
init --> leaf --> workload
2126
end
2227
2328
driver -->|"CreateSandbox<br/>boots via libkrun"| guest
24-
supervisor -.->|"gRPC callback<br/>--grpc-endpoint"| gateway
29+
supervisor <-->|"authenticated RFC 0012<br/>over virtio-vsock"| leaf
2530
26-
client["openshell-cli"] -->|"SSH proxy<br/>127.0.0.1:&lt;port&gt;"| supervisor
31+
client["openshell-cli"] -->|"connect / exec / forward"| gateway
2732
client -->|"CreateSandbox / Watch"| gateway
2833
```
2934

30-
Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside the VM. gvproxy exposes a single inbound SSH port (`host:<allocated>` → `guest:2222`) and provides virtio-net egress.
35+
The control role owns gateway credentials, admitted policy, provider resolution, middleware, the network proxy, and relay registration. The boundary role receives no gateway JWT or mTLS key. It receives a driver-authored boundary token and only the policy/environment state needed to launch the workload.
36+
37+
VM-specific RFC 0012 code under `src/isolation/` only chooses the vsock transport and binds immutable VM generation and image claims into the protected guest config and host descriptor. Lifecycle, authentication, process control, binary identity, forwarding, and streaming come from `openshell-isolation-interface` and `openshell-sandbox`.
3138

3239
## Quick start (recommended)
3340

3441
```shell
3542
mise run gateway:vm
3643
```
3744

38-
First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached.
45+
First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the portable Linux guest leaf plus its trusted helper runtime. The development task also builds the native host supervisor. Subsequent runs are cached.
3946

4047
By default `mise run gateway:vm`:
4148

@@ -96,13 +103,13 @@ rm -rf "${XDG_CONFIG_HOME:-$HOME/.config}/openshell/gateways/vm-dev"
96103
If you want to drive the launch yourself instead of using `mise run gateway:vm` (i.e. `tasks/scripts/gateway-vm.sh`):
97104
98105
```shell
99-
# 1. Stage runtime artifacts + supervisor bundle into target/vm-runtime-compressed/
106+
# 1. Stage runtime artifacts + guest process leaf into target/vm-runtime-compressed/
100107
mise run vm:setup
101-
mise run vm:supervisor # if openshell-sandbox.zst is not already present
108+
mise run vm:supervisor # builds the Linux guest leaf and trusted helper runtime
102109

103-
# 2. Build both binaries with the staged artifacts embedded
110+
# 2. Build gateway, native host supervisor, and driver
104111
OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \
105-
cargo build -p openshell-server -p openshell-driver-vm
112+
cargo build -p openshell-server -p openshell-sandbox -p openshell-driver-vm
106113

107114
# 3. macOS only: codesign the driver for Hypervisor.framework
108115
codesign \
@@ -121,7 +128,7 @@ disable_tls = true
121128
122129
[openshell.drivers.vm]
123130
default_image = "<compatible-image>"
124-
grpc_endpoint = "http://host.containers.internal:18081"
131+
grpc_endpoint = "http://127.0.0.1:18081"
125132
driver_dir = "$PWD/target/debug"
126133
state_dir = "/tmp/openshell-vm-driver-$USER-vm-dev"
127134
EOF
@@ -142,7 +149,7 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr
142149
143150
| Configuration key | Default | Purpose |
144151
|---|---|---|
145-
| `grpc_endpoint` | empty | Required. URL the sandbox guest dials to reach the gateway. Use `http://host.containers.internal:<port>` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:<port>` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. |
152+
| `grpc_endpoint` | empty | Required. URL the native host supervisor uses to reach the gateway. Host loopback such as `http://127.0.0.1:<port>` is valid. This endpoint is never sent into the VM. |
146153
| `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. |
147154
| `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. |
148155
| `default_image` | OpenShell base image | Sandbox image used when a create request omits one. |
@@ -151,9 +158,9 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr
151158
| `mem_mib` | `2048` | Memory per sandbox, in MiB. |
152159
| `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. |
153160
| `krun_log_level` | `1` | libkrun verbosity (0-5). |
154-
| `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. |
155-
| `guest_tls_cert` | unset | Guest client certificate. |
156-
| `guest_tls_key` | unset | Guest client private key. |
161+
| `guest_tls_ca` | unset | Historical key name for the host supervisor's gateway CA certificate. Required when `grpc_endpoint` uses `https://`; never copied into the guest. |
162+
| `guest_tls_cert` | unset | Historical key name for the host supervisor's client certificate; never copied into the guest. |
163+
| `guest_tls_key` | unset | Historical key name for the host supervisor's client private key; never copied into the guest. |
157164
158165
See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface.
159166
@@ -225,14 +232,17 @@ RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \
225232
```
226233
227234
The VM guest's serial console is appended to `<state-dir>/<sandbox-id>/console.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `<state-dir>/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions and removes same-owner stale sockets. On clean shutdown, the gateway sends the managed driver `SIGTERM`, waits up to five seconds for it to flush telemetry and exit, then force-kills it if necessary and removes the socket. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development.
235+
The VM serial console is appended to `<state-dir>/sandboxes/<sandbox-id>/rootfs-console.log`. Host-supervisor stdout and stderr are written beside it as `supervisor.log` and `supervisor.err.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `<state-dir>/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions, removes same-owner stale sockets, and the gateway removes the socket on clean shutdown via `ManagedDriverProcess::drop`. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development.
228236
229237
## Host-side nftables rules
230238
231-
The VM driver creates a per-VM nftables table on the host (`openshell_vm_vmtap_<id>`) with three chains. These rules serve two purposes: NAT infrastructure (required for VM connectivity) and defense-in-depth host isolation. Primary security enforcement — proxy-only egress and bypass detection — is handled by the sandbox supervisor's own nftables rules inside the VM guest.
239+
This section applies to the QEMU/VFIO path, which uses a host TAP device. The normal libkrun path uses gvproxy and needs KVM access but does not require `CAP_NET_ADMIN`. The host supervisor performs primary policy enforcement and receives guest-originated connections through the authenticated network-mediation stream.
240+
241+
The QEMU path creates a per-VM nftables table on the host (`openshell_vm_vmtap_<id>`) with three chains for NAT infrastructure and defense-in-depth host isolation.
232242
233243
**`postrouting` (NAT):** Masquerades outbound VM traffic so it can be routed from the VM's private subnet to the external network. This chain handles forwarded traffic (VM → internet), not traffic destined for the host.
234244
235-
**`forward` (defense-in-depth):** Accepts all outbound traffic from the VM (security enforcement happens guest-side) and accepts established/related response traffic back to the VM. Drops unsolicited inbound connections to the VM from the broader network. This chain handles forwarded traffic only — packets transiting the host between the TAP interface and other interfaces.
245+
**`forward` (defense-in-depth):** Accepts outbound traffic from the VM and established/related response traffic back to the VM. Drops unsolicited inbound connections to the VM from the broader network. This chain handles forwarded traffic only — packets transiting the host between the TAP interface and other interfaces.
236246
237247
**`input` (defense-in-depth):** Accepts traffic from the VM to the gateway port on the host. Drops all other traffic from the VM destined for the host itself. This limits what a compromised guest can reach on the host to the gateway service only.
238248
@@ -250,9 +260,9 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical
250260
- macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM
251261
- Rust toolchain
252262
- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection
253-
- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch):
263+
- Guest-leaf cross-compile toolchain (needed on macOS, and on Linux when host arch differs from the guest):
254264
- Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest)
255-
- `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary.
265+
- `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the Linux `openshell-sandbox --mode=boundary` binary.
256266
- [mise](https://mise.jdx.dev/) task runner
257267
- Docker or Podman socket on the local CLI/gateway host when using
258268
`openshell sandbox create --from ./Dockerfile` or `--from ./dir`; the CLI
@@ -283,11 +293,12 @@ The RPM gateway package is configured for the Podman driver.
283293
284294
On Apple Silicon macOS, `install.sh` stages the generated `openshell.rb`
285295
formula from the selected release in the `nvidia/openshell` Homebrew tap.
286-
Homebrew installs `openshell`, `openshell-gateway`, and
287-
`openshell-driver-vm`, ad-hoc signs the driver with the Hypervisor entitlement
288-
in `post_install`, and owns the `brew services` gateway lifecycle. The service
289-
also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless
290-
the user explicitly overrides it.
296+
Homebrew installs `openshell`, `openshell-gateway`, `openshell-driver-vm`, and
297+
the native `openshell-sandbox` host supervisor beside the driver. It ad-hoc
298+
signs the driver with the Hypervisor entitlement in `post_install` and owns the
299+
`brew services` gateway lifecycle. The service also leaves `OPENSHELL_DRIVERS`
300+
unset so driver choice remains automatic unless the user explicitly overrides
301+
it.
291302
292303
## TODOs
293304

‎crates/openshell-driver-vm/build.rs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ use std::path::{Path, PathBuf};
1010
use std::{env, fs};
1111

1212
fn main() {
13+
if env::var_os("CARGO_FEATURE_COMPUTE_DRIVER").is_none() {
14+
return;
15+
}
16+
1317
println!("cargo:rerun-if-env-changed=OPENSHELL_VM_RUNTIME_COMPRESSED_DIR");
1418

1519
if let Ok(dir) = env::var("OPENSHELL_VM_RUNTIME_COMPRESSED_DIR") {
@@ -38,7 +42,14 @@ fn main() {
3842
println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}");
3943
generate_stub_resources(
4044
&out_dir,
41-
&["libkrun", "libkrunfw", "openshell-sandbox.zst", "umoci.zst"],
45+
&[
46+
"libkrun",
47+
"libkrunfw",
48+
"gvproxy.zst",
49+
"openshell-sandbox.zst",
50+
"openshell-runtime.tar.zst",
51+
"umoci.zst",
52+
],
4253
);
4354
return;
4455
}
@@ -56,6 +67,7 @@ fn main() {
5667
&format!("{libkrunfw_name}.zst"),
5768
"gvproxy.zst",
5869
"openshell-sandbox.zst",
70+
"openshell-runtime.tar.zst",
5971
"umoci.zst",
6072
],
6173
);
@@ -79,6 +91,10 @@ fn main() {
7991
"openshell-sandbox.zst".to_string(),
8092
"openshell-sandbox.zst".to_string(),
8193
),
94+
(
95+
"openshell-runtime.tar.zst".to_string(),
96+
"openshell-runtime.tar.zst".to_string(),
97+
),
8298
("umoci.zst".to_string(), "umoci.zst".to_string()),
8399
];
84100

0 commit comments

Comments
 (0)