Skip to content

Commit 021efef

Browse files
committed
feat(supervisor): report shared supervisor health over TCP
experimental-host takes --health-port instead of a Unix health socket, matching experimental-bootstrap. It binds the wildcard health listener after the session-control and runtime listeners and accepts and closes connections while the process runs. The chart probes the named container port health (5502) with native kubelet tcpSocket checks; session control stays on 5501, the Service exposes only runtime and session control, and /run/openshell remains for per-session files. Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent e15089b commit 021efef

6 files changed

Lines changed: 62 additions & 27 deletions

File tree

‎.agents/skills/helm-dev-environment/SKILL.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,8 @@ The gateway chart creates a single-replica shared supervisor Deployment,
7272
Service, and NetworkPolicy in the release namespace, regardless of
7373
shared/managed/operator workspace mode, and the certgen hook issues its
7474
session-control Secrets. New sandbox generations run as sessions in that
75-
process. Build the gateway, supervisor, and sandbox images from the same branch:
75+
process. Its process-health `tcpSocket` probes use container port `health`
76+
(5502); session control stays on 5501. Build the gateway, supervisor, and sandbox images from the same branch:
7677
all three must support the routed runtime connection. `workspace` is reserved
7778
and rejected. Omit the field to disable it. `mise run
7879
e2e:kubernetes:shared-supervisor` runs the focused e2e

‎crates/openshell-supervisor/README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,9 @@ one process for the Kubernetes gateway topology experiment. It is Unix-only.
119119
- **Runtime connections.** One routed listener (`--runtime-listen`) accepts every
120120
session's runtime-initiated Sandbox Protocol connections; see
121121
`openshell-sandbox-backend` for the routing preface.
122-
- **Health.** `--health-socket` serves the existing `health --socket` check and
122+
- **Health.** `--health-port` (5502 in the chart) binds a wildcard TCP listener
123+
after the session-control and runtime listeners. It accepts and closes
124+
connections for kubelet `tcpSocket` probes, keeps no filesystem state, and
123125
reports process health only. Session readiness is reported per session over
124126
session control. SIGTERM and SIGINT stop every session, which ends their
125127
workloads' supervision.

‎crates/openshell-supervisor/src/experimental.rs‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@ use std::path::PathBuf;
1212
#[command(name = "openshell-supervisor experimental-host")]
1313
#[command(about = "Supervise the sandboxes a gateway registers with this process")]
1414
struct Args {
15-
/// Private socket reporting process health, not session readiness.
15+
/// TCP port accepting connections while the process runs; reports process
16+
/// health, not session readiness.
1617
#[arg(long)]
17-
health_socket: PathBuf,
18+
health_port: u16,
1819

1920
/// Gateway gRPC endpoint; each session authenticates with its own credentials.
2021
#[arg(long, env = openshell_core::sandbox_env::ENDPOINT)]
@@ -113,13 +114,10 @@ fn run_platform(args: Args) -> Result<()> {
113114
.into_diagnostic()
114115
.wrap_err("bind session-control listener")?;
115116

116-
// The private emptyDir survives a container restart after an unclean exit.
117-
match std::fs::remove_file(&args.health_socket) {
118-
Ok(()) => {}
119-
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
120-
Err(error) => return Err(error).into_diagnostic(),
121-
}
122-
let health = tokio::net::UnixListener::bind(&args.health_socket).into_diagnostic()?;
117+
let health = openshell_supervisor::bind_readiness_tcp(args.health_port)
118+
.and_then(tokio::net::TcpListener::from_std)
119+
.into_diagnostic()
120+
.wrap_err("bind health listener")?;
123121
tracing::info!(
124122
runtime = %host.runtime_addr(),
125123
control = %args.session_control_listen,
@@ -135,24 +133,25 @@ fn run_platform(args: Args) -> Result<()> {
135133
let _ = stopped.wait_for(|stop| *stop).await;
136134
},
137135
));
138-
let result = loop {
136+
loop {
139137
tokio::select! {
138+
// A connection is the whole check; nothing is exchanged.
140139
result = health.accept() => {
141140
if let Err(error) = result {
142-
break Err(error).into_diagnostic();
141+
tracing::warn!(%error, "health accept failed; retrying");
142+
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
143143
}
144144
}
145-
_ = terminate.recv() => break Ok(()),
146-
_ = interrupt.recv() => break Ok(()),
145+
_ = terminate.recv() => break,
146+
_ = interrupt.recv() => break,
147147
}
148-
};
148+
}
149149
drop(health);
150-
let _ = std::fs::remove_file(&args.health_socket);
151150
stop.send_replace(true);
152151
let _ = serve.await;
153152
// Sandboxes lose supervision with this process and fail closed.
154153
host.remove_all().await;
155-
result
154+
Ok(())
156155
})
157156
}
158157

‎deploy/helm/openshell/templates/experimental-supervisor.yaml‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ spec:
6060
imagePullPolicy: {{ get (dict "always" "Always" "if_not_present" "IfNotPresent" "never" "Never") (get $kubernetes "supervisor_image_pull_policy" | default "if_not_present") | quote }}
6161
args:
6262
- experimental-host
63-
- --health-socket
64-
- /run/openshell/health.sock
63+
- --health-port
64+
- "5502"
6565
- --session-control-listen
6666
- 0.0.0.0:{{ get $shared "control_port" }}
6767
- --session-control-tls-dir
@@ -90,6 +90,9 @@ spec:
9090
- name: session-control
9191
containerPort: {{ get $shared "control_port" }}
9292
protocol: TCP
93+
- name: health
94+
containerPort: 5502
95+
protocol: TCP
9396
securityContext:
9497
allowPrivilegeEscalation: false
9598
readOnlyRootFilesystem: true
@@ -104,11 +107,11 @@ spec:
104107
# Process health only; each session reports its own readiness to
105108
# the gateway through the session-control API.
106109
readinessProbe:
107-
exec:
108-
command: [/openshell-supervisor, health, --socket, /run/openshell/health.sock]
110+
tcpSocket:
111+
port: health
109112
livenessProbe:
110-
exec:
111-
command: [/openshell-supervisor, health, --socket, /run/openshell/health.sock]
113+
tcpSocket:
114+
port: health
112115
volumeMounts:
113116
- name: run
114117
mountPath: /run/openshell

‎deploy/helm/openshell/tests/experimental_supervisor_test.yaml‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ tests:
4747
path: spec.template.spec.containers[0].args
4848
value:
4949
- experimental-host
50-
- --health-socket
51-
- /run/openshell/health.sock
50+
- --health-port
51+
- "5502"
5252
- --session-control-listen
5353
- 0.0.0.0:5501
5454
- --session-control-tls-dir
@@ -58,6 +58,35 @@ tests:
5858
- --session-root
5959
- /run/openshell/sessions
6060
documentIndex: 0
61+
- contains:
62+
path: spec.template.spec.containers[0].ports
63+
content:
64+
name: health
65+
containerPort: 5502
66+
protocol: TCP
67+
documentIndex: 0
68+
- equal:
69+
path: spec.template.spec.containers[0].readinessProbe
70+
value:
71+
tcpSocket:
72+
port: health
73+
documentIndex: 0
74+
- equal:
75+
path: spec.template.spec.containers[0].livenessProbe
76+
value:
77+
tcpSocket:
78+
port: health
79+
documentIndex: 0
80+
- contains:
81+
path: spec.template.spec.containers[0].volumeMounts
82+
content:
83+
name: run
84+
mountPath: /run/openshell
85+
documentIndex: 0
86+
- lengthEqual:
87+
path: spec.ports
88+
count: 2
89+
documentIndex: 1
6190
- equal:
6291
path: spec.template.spec.automountServiceAccountToken
6392
value: false

‎skills/debug-openshell-cluster/SKILL.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ Helm creates the Deployment, Service, and NetworkPolicy
5757
`<fullname>-experimental-supervisor` in the gateway namespace for every
5858
workspace mode. New sandbox generations run as sessions in that one process, so
5959
there are no per-sandbox supervisor Pods or `os-boundary-<id>` Services for
60-
them. Its probes report process health, not sandbox readiness.
60+
them. Its TCP probes on container port `health` (5502) report process health,
61+
not sandbox readiness; session control keeps port 5501.
6162

6263
- A shared generation's Sandbox resource carries
6364
`openshell.ai/sandbox-runtime-supervisor-topology: gateway` and its session

0 commit comments

Comments
 (0)