Skip to content

Commit 1e9ee47

Browse files
authored
fix(server): suppress expected session errors on shutdown (#2994)
Ctrl-C closes gateway listeners while supervisor sessions are still current, causing benign HTTP/2 broken pipes to be logged as warnings. Track gateway shutdown so recognized transport closes are treated as expected. Signed-off-by: Kris Hicks <khicks@nvidia.com>
1 parent 197b413 commit 1e9ee47

2 files changed

Lines changed: 36 additions & 2 deletions

File tree

crates/openshell-server/src/lib.rs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,10 @@ use std::net::SocketAddr;
6262
use std::path::Path;
6363
#[cfg(test)]
6464
use std::sync::LazyLock;
65-
use std::sync::{Arc, Mutex};
65+
use std::sync::{
66+
Arc, Mutex,
67+
atomic::{AtomicBool, Ordering},
68+
};
6669
use std::time::{Duration, SystemTime, UNIX_EPOCH};
6770
use tokio::net::{TcpListener, TcpStream};
6871
use tokio::sync::watch;
@@ -295,6 +298,10 @@ pub struct ServerState {
295298
/// query session state to surface supervisor readiness.
296299
pub supervisor_sessions: Arc<supervisor_session::SupervisorSessionRegistry>,
297300

301+
/// Set once graceful gateway shutdown begins so stream handlers can
302+
/// distinguish expected transport closes from runtime failures.
303+
pub(crate) gateway_shutting_down: AtomicBool,
304+
298305
/// Validated built-in and operator-registered supervisor middleware.
299306
pub middleware_registry: Arc<MiddlewareRegistry>,
300307

@@ -411,6 +418,7 @@ impl ServerState {
411418
ssh_connections_by_sandbox: Mutex::new(HashMap::new()),
412419
settings_mutex: tokio::sync::Mutex::new(()),
413420
supervisor_sessions,
421+
gateway_shutting_down: AtomicBool::new(false),
414422
extension_mint_limiter: auth::extension_mint_limit::ExtensionMintLimiter::default(),
415423
middleware_registry: Arc::new(MiddlewareRegistry::default()),
416424
oidc_cache,
@@ -802,6 +810,7 @@ pub(crate) async fn run_server(
802810

803811
shutdown_signal().await;
804812
info!("Shutdown signal received; stopping gateway");
813+
state.gateway_shutting_down.store(true, Ordering::Release);
805814
let _ = shutdown_tx.send(true);
806815

807816
for task in listener_tasks {

crates/openshell-server/src/supervisor_session.rs

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::collections::HashMap;
55
use std::pin::Pin;
6+
use std::sync::atomic::Ordering;
67
use std::sync::{Arc, Mutex};
78
use std::time::{Duration, Instant};
89

@@ -666,9 +667,23 @@ async fn expected_transport_close_during_session_teardown(
666667
let session_no_longer_current = !state
667668
.supervisor_sessions
668669
.is_current_session(sandbox_id, session_id);
670+
expected_transport_close_during_session_state(
671+
status,
672+
state.gateway_shutting_down.load(Ordering::Acquire),
673+
session_no_longer_current,
674+
sandbox_is_terminating_or_gone(state, sandbox_id).await,
675+
)
676+
}
677+
678+
fn expected_transport_close_during_session_state(
679+
status: &Status,
680+
gateway_shutting_down: bool,
681+
session_no_longer_current: bool,
682+
sandbox_terminating_or_gone: bool,
683+
) -> bool {
669684
expected_transport_close_during_shutdown(
670685
status,
671-
session_no_longer_current || sandbox_is_terminating_or_gone(state, sandbox_id).await,
686+
gateway_shutting_down || session_no_longer_current || sandbox_terminating_or_gone,
672687
)
673688
}
674689

@@ -1447,6 +1462,16 @@ mod tests {
14471462
assert!(!expected_transport_close_during_shutdown(&status, true));
14481463
}
14491464

1465+
#[test]
1466+
fn gateway_shutdown_makes_session_transport_close_nonfatal() {
1467+
let status =
1468+
Status::unknown("h2 protocol error: error reading a body from connection: broken pipe");
1469+
1470+
assert!(expected_transport_close_during_session_state(
1471+
&status, true, false, false,
1472+
));
1473+
}
1474+
14501475
#[test]
14511476
fn sandbox_proto_terminating_detects_deleting_phase() {
14521477
let mut sandbox = sandbox_record("sbx-1", "sandbox-one");

0 commit comments

Comments
 (0)