Skip to content

Commit daeba51

Browse files
committed
refactor(supervisor): scope gateway credentials and network state to a session
The supervisor kept its gateway bearer, Sandbox Protocol bearer, and renewal task in process-wide slots, and the network supervisor read provider token grants, proxy CA files, and the gateway identity from process environment. A second sandbox in the same process would have replaced or rotated the first sandbox's credentials, reused its provider access tokens, and exposed its denials through policy.local. GatewayCredentials now holds one session's gateway and Sandbox Protocol credentials and their renewal; every gateway client receives a GatewayEndpoint that pairs the URL with them, including log push, ConnectSupervisor and its redirects, and extension-credential rotation. run_networking takes a NetworkSession with the sandbox identity, the session's gateway access, its Workload API socket, its policy.local denial source, and its proxy CA files. Provider token grants use a cache and HTTP client owned by the session and fail closed without a Workload API socket. run_session supervises one generation from an explicit SessionConfig and owner-provided SessionControls. run_sandbox_with_backend remains the dedicated entry point: it keeps the process-wide OCSF context, reads the same environment, binds readiness endpoints to session readiness, and stops on SIGTERM or SIGINT. The binary shares one set of credentials between log push and supervision, as before. Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent 2cbf1d5 commit daeba51

19 files changed

Lines changed: 1429 additions & 672 deletions

File tree

‎crates/openshell-core/src/grpc_client.rs‎

Lines changed: 334 additions & 178 deletions
Large diffs are not rendered by default.

‎crates/openshell-driver-mxc/src/driver.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -870,7 +870,7 @@ async fn run_lifecycle(
870870
binary_path: host_proxy_binary_path(&sandbox_config),
871871
sandbox_id: Some(sandbox_id.clone()),
872872
sandbox_name: Some(sandbox_name.clone()),
873-
openshell_endpoint: None,
873+
gateway: None,
874874
provider_credentials: None,
875875
agent_proposals: openshell_core::proposals::AgentProposals::default(),
876876
denial_tx: None,

‎crates/openshell-supervisor-network/README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,15 @@ allocator implementation belongs to the VM crate. The VM dependency disables
1414
default features, so this instrumentation does not
1515
require the compute runtime or Unix-only sandbox library. It is a development
1616
dependency only.
17+
18+
## Session inputs
19+
20+
`run_networking` takes a `NetworkSession` with the sandbox identity, the
21+
session's `GatewayEndpoint`, the SPIFFE Workload API socket for provider token
22+
grants, the `policy.local` denial source, and proxy CA files. None of these
23+
are read from the process environment, so one process can run several
24+
sessions. Each session owns its provider token cache and HTTP client; a token
25+
acquired for one sandbox is never served to another. Without a Workload API
26+
socket, token grants fail closed. A process hosting several sessions serves
27+
`/v1/denials` from a per-session `SessionDenialLog` instead of the shared
28+
shorthand log files.

‎crates/openshell-supervisor-network/src/host.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ pub struct HostProxyConfig {
4848
pub binary_path: PathBuf,
4949
pub sandbox_id: Option<String>,
5050
pub sandbox_name: Option<String>,
51-
pub openshell_endpoint: Option<String>,
51+
/// Gateway access with this sandbox's credentials.
52+
pub gateway: Option<openshell_core::grpc_client::GatewayEndpoint>,
5253
pub provider_credentials: Option<ProviderCredentialState>,
5354
/// Shared feature state for the policy.local agent proposal surface.
5455
pub agent_proposals: AgentProposals,
@@ -99,7 +100,7 @@ pub async fn start_host_proxy(config: HostProxyConfig) -> Result<HostProxyHandle
99100
let (_workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new());
100101
let policy_local_ctx = Arc::new(PolicyLocalContext::new(
101102
Some(config.policy.clone()),
102-
config.openshell_endpoint.clone(),
103+
config.gateway.clone(),
103104
config
104105
.sandbox_name
105106
.clone()

‎crates/openshell-supervisor-network/src/l7/token_grant_injection.rs‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -53,18 +53,26 @@ pub trait TokenGrantResolver: Send + Sync {
5353
) -> Pin<Box<dyn Future<Output = Result<String, TokenGrantFailure>> + Send + 'a>>;
5454
}
5555

56-
/// Resolves provider tokens using the supervisor's authenticated grant flow.
57-
#[derive(Default)]
58-
pub struct SpiffeTokenGrantResolver;
56+
/// Resolves provider tokens through one session's authenticated grant flow.
57+
pub struct SpiffeTokenGrantResolver {
58+
grants: crate::token_grant::SessionTokenGrants,
59+
}
60+
61+
impl SpiffeTokenGrantResolver {
62+
#[must_use]
63+
pub fn new(grants: crate::token_grant::SessionTokenGrants) -> Self {
64+
Self { grants }
65+
}
66+
}
5967

6068
impl TokenGrantResolver for SpiffeTokenGrantResolver {
6169
fn obtain<'a>(
6270
&'a self,
6371
request: TokenGrantRequest<'a>,
6472
) -> Pin<Box<dyn Future<Output = Result<String, TokenGrantFailure>> + Send + 'a>> {
6573
Box::pin(async move {
66-
crate::token_grant::obtain_provider_token(
67-
crate::token_grant::ObtainProviderTokenRequest {
74+
self.grants
75+
.obtain_provider_token(crate::token_grant::ObtainProviderTokenRequest {
6876
provider_name: request.provider_key,
6977
token_endpoint: request.token_endpoint,
7078
jwt_svid_audience: request.jwt_svid_audience,
@@ -75,18 +83,12 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver {
7583
grant_type: request.grant_type,
7684
requested_token_type: request.requested_token_type,
7785
request_id: request.request_id,
78-
},
79-
)
80-
.await
86+
})
87+
.await
8188
})
8289
}
8390
}
8491

85-
/// Construct the production SPIFFE-backed resolver.
86-
pub fn default_resolver() -> Arc<dyn TokenGrantResolver> {
87-
Arc::new(SpiffeTokenGrantResolver)
88-
}
89-
9092
/// A refused request: the grant failure and the credential whose grant failed.
9193
///
9294
/// The credential is the `provider:credential` name pair of the selected

‎crates/openshell-supervisor-network/src/policy_local.rs‎

Lines changed: 144 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@
33

44
//! Sandbox-local policy advisor HTTP API.
55
6+
use crate::l7::token_grant_injection::TokenGrantResolver;
67
use miette::{IntoDiagnostic, Result};
8+
use openshell_core::grpc_client::GatewayEndpoint;
79
use openshell_core::proposals::AgentProposals;
810
use openshell_core::proto::{
911
L7Allow, L7DenyRule, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, PolicyChunk,
@@ -82,50 +84,153 @@ const SHORTHAND_LOG_PREFIX: &str = "openshell.";
8284
/// log entry (very long URL path, etc.) cannot blow up the response.
8385
const MAX_DENIAL_LINE_BYTES: usize = 4096;
8486

85-
#[derive(Debug)]
87+
/// Most recent denial lines retained for one hosted session.
88+
const MAX_SESSION_DENIAL_LINES: usize = 256;
89+
90+
/// Where `/v1/denials` reads this sandbox's recent denials.
91+
#[derive(Clone, Debug)]
92+
pub enum DenialLogSource {
93+
/// Shorthand log files written by a supervisor process that serves one
94+
/// sandbox.
95+
ShorthandFiles(PathBuf),
96+
/// Denials recorded for this session alone. A process hosting several
97+
/// sandboxes cannot expose its shared log files to any one of them.
98+
Session(SessionDenialLog),
99+
}
100+
101+
impl Default for DenialLogSource {
102+
fn default() -> Self {
103+
Self::ShorthandFiles(PathBuf::from(LOG_DIR))
104+
}
105+
}
106+
107+
/// Bounded recent OCSF denial lines recorded for one hosted session.
108+
#[derive(Clone, Default)]
109+
pub struct SessionDenialLog {
110+
lines: Arc<std::sync::Mutex<std::collections::VecDeque<String>>>,
111+
}
112+
113+
impl std::fmt::Debug for SessionDenialLog {
114+
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
115+
let retained = self
116+
.lines
117+
.lock()
118+
.unwrap_or_else(std::sync::PoisonError::into_inner)
119+
.len();
120+
formatter
121+
.debug_struct("SessionDenialLog")
122+
.field("retained", &retained)
123+
.finish()
124+
}
125+
}
126+
127+
impl SessionDenialLog {
128+
/// Retain `line` when it is a rendered OCSF denial; ignore anything else.
129+
pub fn record(&self, line: &str) {
130+
let line = line.trim_end();
131+
if !is_ocsf_denial_line(line) {
132+
return;
133+
}
134+
let mut lines = self
135+
.lines
136+
.lock()
137+
.unwrap_or_else(std::sync::PoisonError::into_inner);
138+
if lines.len() == MAX_SESSION_DENIAL_LINES {
139+
lines.pop_front();
140+
}
141+
lines.push_back(line.to_string());
142+
}
143+
144+
fn recent(&self, limit: usize) -> Vec<String> {
145+
self.lines
146+
.lock()
147+
.unwrap_or_else(std::sync::PoisonError::into_inner)
148+
.iter()
149+
.rev()
150+
.take(limit)
151+
.map(|line| surfaced_denial_line(line))
152+
.collect()
153+
}
154+
}
155+
86156
pub struct PolicyLocalContext {
87157
current_policy: Arc<RwLock<Option<Arc<ProtoSandboxPolicy>>>>,
88158
agent_proposals: AgentProposals,
89-
gateway_endpoint: Option<String>,
159+
gateway: Option<GatewayEndpoint>,
90160
sandbox_name: Option<String>,
91-
shorthand_log_dir: PathBuf,
161+
denials: DenialLogSource,
162+
token_grants: Option<Arc<dyn TokenGrantResolver>>,
92163
workspace_rx: tokio::sync::watch::Receiver<String>,
93164
}
94165

166+
impl std::fmt::Debug for PolicyLocalContext {
167+
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
168+
formatter
169+
.debug_struct("PolicyLocalContext")
170+
.field("gateway", &self.gateway)
171+
.field("sandbox_name", &self.sandbox_name)
172+
.field("denials", &self.denials)
173+
.field("token_grants", &self.token_grants.is_some())
174+
.finish_non_exhaustive()
175+
}
176+
}
177+
95178
impl PolicyLocalContext {
96179
pub fn new(
97180
current_policy: Option<ProtoSandboxPolicy>,
98-
gateway_endpoint: Option<String>,
181+
gateway: Option<GatewayEndpoint>,
99182
sandbox_name: Option<String>,
100183
agent_proposals: AgentProposals,
101184
workspace_rx: tokio::sync::watch::Receiver<String>,
102185
) -> Self {
103-
Self::with_log_dir(
104-
current_policy,
105-
gateway_endpoint,
106-
sandbox_name,
186+
Self {
187+
current_policy: Arc::new(RwLock::new(current_policy.map(Arc::new))),
107188
agent_proposals,
108-
PathBuf::from(LOG_DIR),
189+
gateway,
190+
sandbox_name,
191+
denials: DenialLogSource::default(),
192+
token_grants: None,
109193
workspace_rx,
110-
)
194+
}
111195
}
112196

197+
#[cfg(test)]
113198
fn with_log_dir(
114199
current_policy: Option<ProtoSandboxPolicy>,
115-
gateway_endpoint: Option<String>,
200+
gateway: Option<GatewayEndpoint>,
116201
sandbox_name: Option<String>,
117202
agent_proposals: AgentProposals,
118203
shorthand_log_dir: PathBuf,
119204
workspace_rx: tokio::sync::watch::Receiver<String>,
120205
) -> Self {
121-
Self {
122-
current_policy: Arc::new(RwLock::new(current_policy.map(Arc::new))),
123-
agent_proposals,
124-
gateway_endpoint,
206+
Self::new(
207+
current_policy,
208+
gateway,
125209
sandbox_name,
126-
shorthand_log_dir,
210+
agent_proposals,
127211
workspace_rx,
128-
}
212+
)
213+
.with_denial_log(DenialLogSource::ShorthandFiles(shorthand_log_dir))
214+
}
215+
216+
/// Serve `/v1/denials` from `denials` instead of the process log files.
217+
#[must_use]
218+
pub fn with_denial_log(mut self, denials: DenialLogSource) -> Self {
219+
self.denials = denials;
220+
self
221+
}
222+
223+
/// Acquire provider token grants for this sandbox's HTTP relays.
224+
#[must_use]
225+
pub fn with_token_grants(mut self, resolver: Arc<dyn TokenGrantResolver>) -> Self {
226+
self.token_grants = Some(resolver);
227+
self
228+
}
229+
230+
/// The session's provider token grant resolver, absent when grants are
231+
/// unavailable and must fail closed.
232+
pub(crate) fn token_grant_resolver(&self) -> Option<Arc<dyn TokenGrantResolver>> {
233+
self.token_grants.clone()
129234
}
130235

131236
pub async fn set_current_policy(&self, policy: ProtoSandboxPolicy) {
@@ -318,7 +423,18 @@ async fn recent_denials_response(
318423
query: &str,
319424
) -> (u16, serde_json::Value) {
320425
let limit = parse_last_query(query).unwrap_or(DEFAULT_DENIALS_LIMIT);
321-
let log_dir = ctx.shorthand_log_dir.clone();
426+
let log_dir = match &ctx.denials {
427+
DenialLogSource::ShorthandFiles(log_dir) => log_dir.clone(),
428+
DenialLogSource::Session(log) => {
429+
return (
430+
200,
431+
serde_json::json!({
432+
"denials": log.recent(limit),
433+
"log_available": true,
434+
}),
435+
);
436+
}
437+
};
322438

323439
// Distinguish "shorthand log exists and no denials happened" from "no log
324440
// file yet, so we have nothing to read." Without this flag the agent sees
@@ -398,9 +514,7 @@ fn read_recent_denial_lines(log_dir: &Path, limit: usize) -> Vec<String> {
398514
// proxy.rs FORWARD path so the on-disk shorthand log itself is
399515
// clean — tracked separately). Redact first so truncation cannot
400516
// slice mid-secret.
401-
let redacted = redact_query_strings(line);
402-
let surfaced = truncate_at_char_boundary(&redacted, MAX_DENIAL_LINE_BYTES);
403-
lines.push(surfaced);
517+
lines.push(surfaced_denial_line(line));
404518
if lines.len() >= limit {
405519
return lines;
406520
}
@@ -409,6 +523,11 @@ fn read_recent_denial_lines(log_dir: &Path, limit: usize) -> Vec<String> {
409523
lines
410524
}
411525

526+
fn surfaced_denial_line(line: &str) -> String {
527+
let redacted = redact_query_strings(line);
528+
truncate_at_char_boundary(&redacted, MAX_DENIAL_LINE_BYTES)
529+
}
530+
412531
/// Replace any `?<query>` substring with `?[redacted]` to keep query-string
413532
/// secrets out of the agent's view. Walks per Unicode scalar value so multi-byte
414533
/// content is safe. A query is everything from `?` until the next whitespace or
@@ -486,7 +605,7 @@ fn collect_shorthand_log_files(log_dir: &Path, max_files: usize) -> std::io::Res
486605
}
487606

488607
async fn submit_proposal(ctx: &PolicyLocalContext, body: &[u8]) -> (u16, serde_json::Value) {
489-
let Some(endpoint) = ctx.gateway_endpoint.as_deref() else {
608+
let Some(gateway) = ctx.gateway.as_ref() else {
490609
return (
491610
503,
492611
serde_json::json!({
@@ -526,7 +645,7 @@ async fn submit_proposal(ctx: &PolicyLocalContext, body: &[u8]) -> (u16, serde_j
526645
Err(error) => return (400, error_payload("invalid_proposal", error)),
527646
};
528647

529-
let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await {
648+
let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(gateway).await {
530649
Ok(client) => {
531650
client.set_workspace(workspace);
532651
client
@@ -956,7 +1075,7 @@ struct LookupSession<'a> {
9561075
async fn open_lookup_session(
9571076
ctx: &PolicyLocalContext,
9581077
) -> std::result::Result<LookupSession<'_>, (u16, serde_json::Value)> {
959-
let endpoint = ctx.gateway_endpoint.as_deref().ok_or_else(|| {
1078+
let gateway = ctx.gateway.as_ref().ok_or_else(|| {
9601079
(
9611080
503,
9621081
error_payload(
@@ -989,7 +1108,7 @@ async fn open_lookup_session(
9891108
),
9901109
));
9911110
}
992-
let client = openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint)
1111+
let client = openshell_core::grpc_client::CachedOpenShellClient::connect(gateway)
9931112
.await
9941113
.map_err(|e| (502, error_payload("gateway_connect_failed", e.to_string())))?;
9951114
client.set_workspace(workspace);

0 commit comments

Comments
 (0)