Skip to content

Commit 52aac37

Browse files
authored
fix(network): honor HTTP response connection closure (NVIDIA#3581)
* fix(network): honor HTTP response connection closure Signed-off-by: Drew Newberry <anewberry@nvidia.com> * test(network): fix EOF fixture socket setup Signed-off-by: Drew Newberry <anewberry@nvidia.com> * test(network): keep pipeline probe response reusable Signed-off-by: Drew Newberry <anewberry@nvidia.com> --------- Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent 9563240 commit 52aac37

7 files changed

Lines changed: 336 additions & 32 deletions

File tree

‎architecture/sandbox.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,12 @@ the shared raw byte relay after the existing adapter gates. Forward HTTP retains
280280
its guarded single-request relay while sharing authorization, request context,
281281
policy-pinning, and destination boundaries.
282282
Adapter-specific response and OCSF event shapes remain at the protocol boundary.
283+
HTTP response framing and connection persistence are separate decisions. After
284+
forwarding a complete closing response (explicit `Connection: close` or HTTP/1.0
285+
without keep-alive), the relay flushes and shuts down downstream writes before
286+
ending the exchange, including TLS close notification. Response middleware
287+
preserves this lifetime rule; persistent responses remain eligible for reuse.
288+
283289
An explicit `protocol: tcp` endpoint with a valid DNS hostname opts into native
284290
DNS and transparent TCP when the selected runtime advertises that substrate.
285291
Hostless `allowed_ips` and literal-IP selectors remain available only to the

‎crates/openshell-supervisor-network/src/l7/rest.rs‎

Lines changed: 124 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3663,7 +3663,7 @@ mod tests {
36633663
const VALID_WS_ACCEPT: &str = "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=";
36643664
const TEXT_OPCODE: u8 = 0x1;
36653665

3666-
#[derive(Clone, Copy)]
3666+
#[derive(Debug, Clone, Copy)]
36673667
enum ResponseRelayScript {
36683668
HeadersOnly,
36693669
WholeBody,
@@ -7154,7 +7154,7 @@ mod tests {
71547154
ResponseRelayScript::HeadersOnly,
71557155
)
71567156
.await;
7157-
assert!(matches!(outcome.unwrap(), RelayOutcome::Reusable));
7157+
assert!(matches!(outcome.unwrap(), RelayOutcome::Consumed));
71587158

71597159
let (outcome, delivered) = run_response_middleware_relay(
71607160
b"HTTP/1.1 200 OK\r\n\r\n",
@@ -7931,6 +7931,117 @@ mod tests {
79317931
assert!(received_str.contains("hello"));
79327932
}
79337933

7934+
#[tokio::test]
7935+
async fn response_persistence_is_independent_of_framing_and_middleware() {
7936+
for script in [
7937+
None,
7938+
Some(ResponseRelayScript::HeadersOnly),
7939+
Some(ResponseRelayScript::Stream),
7940+
Some(ResponseRelayScript::WholeBody),
7941+
] {
7942+
for (method, response, closes) in [
7943+
(
7944+
"GET",
7945+
"HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\nhello",
7946+
true,
7947+
),
7948+
(
7949+
"GET",
7950+
"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 5\r\n\r\nhello",
7951+
true,
7952+
),
7953+
(
7954+
"GET",
7955+
"HTTP/1.1 200 OK\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n",
7956+
true,
7957+
),
7958+
("HEAD", "HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\n", true),
7959+
("GET", "HTTP/1.0 204 No Content\r\n\r\n", true),
7960+
(
7961+
"GET",
7962+
"HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n",
7963+
true,
7964+
),
7965+
(
7966+
"GET",
7967+
"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello",
7968+
false,
7969+
),
7970+
(
7971+
"GET",
7972+
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n",
7973+
false,
7974+
),
7975+
(
7976+
"HEAD",
7977+
"HTTP/1.0 200 OK\r\nConnection: keep-alive\r\nContent-Length: 5\r\n\r\n",
7978+
false,
7979+
),
7980+
] {
7981+
// Body-inspecting scripts reject bodiless responses at preflight.
7982+
if (method == "HEAD"
7983+
|| response.contains("204 No Content")
7984+
|| response.contains("304 Not Modified"))
7985+
&& matches!(
7986+
script,
7987+
Some(ResponseRelayScript::Stream | ResponseRelayScript::WholeBody)
7988+
)
7989+
{
7990+
continue;
7991+
}
7992+
let fixture = script.map(response_middleware_fixture);
7993+
let middleware = fixture
7994+
.as_ref()
7995+
.map(|(runner, chain)| response_middleware_context(runner, chain, method));
7996+
let mut upstream = response.as_bytes();
7997+
let (mut reader, mut writer) = tokio::io::duplex(4096);
7998+
let outcome = relay_response(
7999+
method,
8000+
&mut upstream,
8001+
&mut writer,
8002+
RelayResponseOptions::default(),
8003+
middleware,
8004+
)
8005+
.await
8006+
.unwrap();
8007+
assert_eq!(
8008+
matches!(outcome, RelayOutcome::Consumed),
8009+
closes,
8010+
"{response}"
8011+
);
8012+
let mut delivered = Vec::new();
8013+
if closes {
8014+
tokio::time::timeout(
8015+
std::time::Duration::from_secs(2),
8016+
reader.read_to_end(&mut delivered),
8017+
)
8018+
.await
8019+
.unwrap_or_else(|error| panic!("{script:?} {method} {response:?}: {error}"))
8020+
.unwrap();
8021+
} else {
8022+
// A second write demonstrates that persistent output is still open.
8023+
writer.write_all(b"next response").await.unwrap();
8024+
writer.shutdown().await.unwrap();
8025+
reader.read_to_end(&mut delivered).await.unwrap();
8026+
assert!(delivered.ends_with(b"next response"));
8027+
}
8028+
if let Some(script) = script {
8029+
let expected = match script {
8030+
ResponseRelayScript::HeadersOnly => "hello",
8031+
ResponseRelayScript::Stream => "HELLO",
8032+
ResponseRelayScript::WholeBody => "whole:hello",
8033+
_ => unreachable!(),
8034+
};
8035+
if response.ends_with("hello") {
8036+
assert!(String::from_utf8_lossy(&delivered).contains(expected));
8037+
}
8038+
} else {
8039+
assert!(delivered.starts_with(response.as_bytes()));
8040+
}
8041+
}
8042+
}
8043+
}
8044+
79348045
#[tokio::test]
79358046
async fn relay_response_connection_close_with_content_length() {
79368047
let response = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nhello";
@@ -7956,18 +8067,19 @@ mod tests {
79568067
.expect("relay must not deadlock");
79578068

79588069
let outcome = result.expect("relay_response should succeed");
7959-
// With explicit framing, Connection: close is still reported as reusable
7960-
// so the relay loop continues. The *next* upstream write will fail and
7961-
// exit the loop via the normal error path.
7962-
assert!(
7963-
matches!(outcome, RelayOutcome::Reusable),
7964-
"explicit framing keeps loop alive despite Connection: close"
7965-
);
8070+
assert!(matches!(outcome, RelayOutcome::Consumed));
79668071

7967-
client_write.shutdown().await.unwrap();
8072+
// Keep the relay-side stream alive: EOF must come from shutdown,
8073+
// not from dropping the stream or the test closing it manually.
79688074
let mut received = Vec::new();
7969-
client_read.read_to_end(&mut received).await.unwrap();
7970-
assert!(String::from_utf8_lossy(&received).contains("hello"));
8075+
tokio::time::timeout(
8076+
std::time::Duration::from_secs(2),
8077+
client_read.read_to_end(&mut received),
8078+
)
8079+
.await
8080+
.expect("closing response must deliver EOF")
8081+
.unwrap();
8082+
assert_eq!(received, response);
79718083
}
79728084

79738085
#[tokio::test]

‎crates/openshell-supervisor-network/src/l7/rest/http_response.rs‎

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,11 @@ where
178178
))
179179
.await?
180180
{
181-
return Ok(outcome);
181+
return if matches!(outcome, RelayOutcome::Consumed) {
182+
finish_response(client, true).await
183+
} else {
184+
Ok(outcome)
185+
};
182186
}
183187

184188
// Bodiless responses (HEAD, 1xx, 204, 304): forward headers only, skip body
@@ -187,12 +191,7 @@ where
187191
.write_all(&buf[..header_end])
188192
.await
189193
.into_diagnostic()?;
190-
client.flush().await.into_diagnostic()?;
191-
return if server_wants_close {
192-
Ok(RelayOutcome::Consumed)
193-
} else {
194-
Ok(RelayOutcome::Reusable)
195-
};
194+
return finish_response(client, server_wants_close || http_10_closes_by_default).await;
196195
}
197196

198197
// No explicit framing (no Content-Length, no Transfer-Encoding).
@@ -261,13 +260,24 @@ where
261260
"relay_response complete (explicit framing)"
262261
);
263262

264-
// When body framing is explicit (Content-Length / Chunked), always report
265-
// the connection as reusable so the relay loop continues. If the server
266-
// sent `Connection: close`, the *next* upstream write will fail and the
267-
// loop will exit via the normal error path. Exiting early here would
268-
// tear down the CONNECT tunnel before the client can detect the close,
269-
// causing ~30 s retry delays in clients like `gh`.
270-
Ok(RelayOutcome::Reusable)
263+
finish_response(client, server_wants_close || http_10_closes_by_default).await
264+
}
265+
266+
/// Body framing determines when delivery finishes, not whether another
267+
/// request is permitted. Signal EOF (including TLS `close_notify`) before the
268+
/// caller tears down a closing CONNECT tunnel; waiting for another request
269+
/// deadlocks clients that are themselves waiting for EOF.
270+
async fn finish_response<C>(client: &mut C, close: bool) -> Result<RelayOutcome>
271+
where
272+
C: AsyncWrite + Unpin,
273+
{
274+
client.flush().await.into_diagnostic()?;
275+
if close {
276+
client.shutdown().await.into_diagnostic()?;
277+
Ok(RelayOutcome::Consumed)
278+
} else {
279+
Ok(RelayOutcome::Reusable)
280+
}
271281
}
272282

273283
#[allow(clippy::too_many_arguments)]
@@ -758,8 +768,9 @@ where
758768
}
759769
client.flush().await.into_diagnostic()?;
760770
Ok(Some(
761-
if (committed && close_delimited_output)
762-
|| (matches!(body_length, BodyLength::None) && (server_wants_close || event_stream))
771+
if server_wants_close
772+
|| (committed && close_delimited_output)
773+
|| (matches!(body_length, BodyLength::None) && event_stream)
763774
{
764775
RelayOutcome::Consumed
765776
} else {
@@ -827,7 +838,11 @@ where
827838
BodyLength::None => {}
828839
}
829840
client.flush().await.into_diagnostic()?;
830-
Ok(RelayOutcome::Reusable)
841+
Ok(if server_wants_close {
842+
RelayOutcome::Consumed
843+
} else {
844+
RelayOutcome::Reusable
845+
})
831846
}
832847

833848
fn emit_http_response_middleware_invocations(

‎crates/openshell-supervisor-network/src/proxy/relay.rs‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,101 @@ mod tests {
374374
}
375375
}
376376

377+
async fn assert_response_lifecycle<C, P>(mut caller: C, mut client: P)
378+
where
379+
C: AsyncRead + AsyncWrite + Unpin,
380+
P: AsyncRead + AsyncWrite + Unpin + Send,
381+
{
382+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
383+
384+
let (mut upstream, mut server) = tokio::io::duplex(1024);
385+
let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap();
386+
let decision = decision(engine.current_generation());
387+
let request = request_context();
388+
let context = prepare_http_relay(None, &engine, &decision, &request).unwrap();
389+
let persistent = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello";
390+
// Larger than either transport buffer: closing must drain the body.
391+
let body = vec![b'x'; 64 * 1024];
392+
let mut closing = format!(
393+
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
394+
body.len()
395+
)
396+
.into_bytes();
397+
closing.extend_from_slice(&body);
398+
let relay = Box::pin(relay_http_stream(&mut client, &mut upstream, context));
399+
let serve = async {
400+
for response in [persistent.as_slice(), closing.as_slice()] {
401+
let mut request = Vec::new();
402+
while !request.ends_with(b"\r\n\r\n") {
403+
request.push(server.read_u8().await.unwrap());
404+
assert!(request.len() < 4096);
405+
}
406+
assert!(request.starts_with(b"GET / HTTP/1.1\r\n"));
407+
server.write_all(response).await.unwrap();
408+
server.flush().await.unwrap();
409+
}
410+
// Retain the upstream socket: response headers decide persistence.
411+
};
412+
let receive = async {
413+
let request = b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
414+
caller.write_all(request).await.unwrap();
415+
caller.flush().await.unwrap();
416+
let mut first = vec![0; persistent.len()];
417+
caller.read_exact(&mut first).await.unwrap();
418+
assert_eq!(first, persistent);
419+
// A real second exchange verifies reuse, without a timing assertion.
420+
caller.write_all(request).await.unwrap();
421+
caller.flush().await.unwrap();
422+
let mut second = Vec::new();
423+
// TLS must return clean EOF, not UnexpectedEof from a dropped socket.
424+
caller.read_to_end(&mut second).await.unwrap();
425+
assert_eq!(second, closing);
426+
};
427+
tokio::time::timeout(std::time::Duration::from_secs(5), async {
428+
let (result, (), ()) = tokio::join!(relay, serve, receive);
429+
result.unwrap();
430+
})
431+
.await
432+
.expect("response delivery and EOF must not await another request");
433+
}
434+
435+
#[tokio::test]
436+
async fn http_relay_reuses_then_closes_after_complete_response() {
437+
let (caller, client) = tokio::io::duplex(1024);
438+
assert_response_lifecycle(caller, client).await;
439+
}
440+
441+
#[tokio::test]
442+
async fn tls_http_relay_reuses_then_sends_close_notify_after_complete_response() {
443+
use crate::l7::tls::{CertCache, ProxyTlsState, SandboxCa, tls_terminate_client};
444+
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
445+
let ca = SandboxCa::generate().unwrap();
446+
let mut roots = rustls::RootCertStore::empty();
447+
for cert in rustls_pemfile::certs(&mut ca.cert_pem().as_bytes()) {
448+
roots.add(cert.unwrap()).unwrap();
449+
}
450+
let config = Arc::new(
451+
rustls::ClientConfig::builder()
452+
.with_root_certificates(roots)
453+
.with_no_client_auth(),
454+
);
455+
let state = ProxyTlsState::new(CertCache::new(ca), config.clone());
456+
let connector = tokio_rustls::TlsConnector::from(config);
457+
let (caller, client) = tokio::io::duplex(1024);
458+
let (caller, client) = tokio::time::timeout(std::time::Duration::from_secs(5), async {
459+
tokio::join!(
460+
connector.connect(
461+
rustls::pki_types::ServerName::try_from("example.com").unwrap(),
462+
caller
463+
),
464+
tls_terminate_client(client, &state, "example.com"),
465+
)
466+
})
467+
.await
468+
.expect("TLS handshake must complete");
469+
assert_response_lifecycle(caller.unwrap(), client.unwrap()).await;
470+
}
471+
377472
#[test]
378473
fn relay_without_route_pins_l4_decision_generation() {
379474
let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap();

‎docs/reference/policy-schema.mdx‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,11 @@ access selects no preset. Unknown enum numbers are rejected before activation.
267267
- `deny_rules: []` (empty list) is rejected; remove it if no denials are needed.
268268
- `credential_signing` requires a resolvable AWS credential source before a sandbox policy can activate. Use an attached endpoint-bearing profile that declares `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` and covers the signed endpoint, or bind an attached endpointless profile that declares those keys with `credential_binding.provider`.
269269

270+
For proxied HTTP responses, OpenShell completes delivery and signals EOF when
271+
the server sends `Connection: close` or an HTTP/1.0 response without keep-alive.
272+
This also applies to HTTPS and responses processed by middleware. A
273+
`Content-Length` or chunked body does not override the server's close decision.
274+
270275
Credential rewrite recognizes the canonical `openshell:resolve:env:KEY` placeholder form and whole-token provider-shaped aliases such as `provider-OPENSHELL-RESOLVE-ENV-API_TOKEN` when the referenced environment key exists in the configured provider credentials.
271276

272277
Static provider placeholders also require the request host, port, and path to

0 commit comments

Comments
 (0)