Skip to content

Commit 7a50c08

Browse files
authored
fix(cli): stream piped exec stdin beyond gRPC request limit (NVIDIA#3687)
* fix(cli): stream piped exec stdin across gRPC messages Signed-off-by: Drew Newberry <anewberry@nvidia.com> * fix(cli): preserve small exec requests and surface stdin errors Signed-off-by: Drew Newberry <anewberry@nvidia.com> * fix(cli): preserve exec stdin limit across gRPC streaming Signed-off-by: Drew Newberry <anewberry@nvidia.com> --------- Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent 4688061 commit 7a50c08

6 files changed

Lines changed: 311 additions & 70 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎architecture/gateway.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1111,6 +1111,11 @@ as normal EOF. The input and output pumps are owned by the exec operation, so
11111111
timeout or response abandonment cannot leave a detached stdin task behind.
11121112
The pumps share polling fairly, and request processing yields cooperatively even
11131113
for ignored resize messages, so sustained input cannot monopolize the operation.
1114+
The CLI keeps piped input in the unary request when the complete encoded
1115+
request fits the gateway's decoder limit, preserving compatibility with older
1116+
gateways. Input up to the CLI's 4 MiB cap uses this stream with bounded frames
1117+
when the unary message would exceed the decoder limit, with or without a PTY.
1118+
The CLI closes the input side at pipe EOF.
11141119

11151120
Go and TypeScript interactive-exec helpers distinguish process exit from stream
11161121
completion. They consume the final gRPC status before reporting success and retain

‎crates/openshell-cli/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ openshell-tui = { path = "../openshell-tui" }
2424
serde = { workspace = true }
2525
serde_json = { workspace = true }
2626
serde_yml = { workspace = true }
27+
prost = { workspace = true }
2728
prost-types = { workspace = true }
2829

2930
# Async runtime

‎crates/openshell-cli/src/run.rs‎

Lines changed: 152 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ use openshell_core::proto::{
6464
};
6565
use openshell_core::settings;
6666
use openshell_core::{ObjectId, ObjectName, ObjectWorkspace};
67+
use prost::Message;
6768
use std::borrow::Cow;
6869
use std::collections::HashMap;
6970
use std::io::{ErrorKind, IsTerminal, Read, Write};
@@ -1794,16 +1795,15 @@ where
17941795
Ok(())
17951796
}
17961797

1797-
/// Maximum stdin payload size (4 MiB). Prevents the CLI from reading unbounded
1798-
/// data into memory before the server rejects an oversized message.
1799-
const MAX_STDIN_PAYLOAD: usize = 4 * 1024 * 1024;
1800-
18011798
fn local_terminal_size() -> Option<(u32, u32)> {
18021799
crossterm::terminal::size()
18031800
.ok()
18041801
.map(|(cols, rows)| (u32::from(cols), u32::from(rows)))
18051802
}
18061803

1804+
const MAX_EXEC_REQUEST_BYTES: usize = 1024 * 1024;
1805+
const MAX_EXEC_STDIN_BYTES: usize = 4 * 1024 * 1024;
1806+
18071807
/// Execute a command in a running sandbox via gRPC, streaming output to the terminal.
18081808
///
18091809
/// Returns the remote command's exit code, or an error if the event stream
@@ -1846,72 +1846,83 @@ pub async fn sandbox_exec_grpc(
18461846
));
18471847
}
18481848

1849-
// Read stdin if piped (not a TTY), using spawn_blocking to avoid blocking
1850-
// the async runtime. Cap the read at MAX_STDIN_PAYLOAD + 1 so we never
1851-
// buffer more than the limit into memory.
1852-
let stdin_payload = if std::io::stdin().is_terminal() {
1849+
// Resolve TTY mode: explicit --tty / --no-tty wins, otherwise auto-detect.
1850+
let stdin_is_terminal = std::io::stdin().is_terminal();
1851+
let tty = tty_override.unwrap_or_else(|| stdin_is_terminal && std::io::stdout().is_terminal());
1852+
1853+
// Preserve unary exec for small pipes, including older gateways whose
1854+
// interactive RPC closes the SSH channel when stdin reaches EOF. Retain
1855+
// the existing 4 MiB input cap because the supervisor's process stdin
1856+
// queue is unbounded; larger input should use file upload instead.
1857+
let stdin_prefix = if stdin_is_terminal {
18531858
Vec::new()
18541859
} else {
18551860
tokio::task::spawn_blocking(|| {
1856-
let limit = (MAX_STDIN_PAYLOAD + 1) as u64;
1857-
let mut buf = Vec::new();
1861+
let mut prefix = Vec::new();
18581862
std::io::stdin()
1859-
.take(limit)
1860-
.read_to_end(&mut buf)
1863+
.take((MAX_EXEC_STDIN_BYTES + 1) as u64)
1864+
.read_to_end(&mut prefix)
18611865
.into_diagnostic()?;
1862-
if buf.len() > MAX_STDIN_PAYLOAD {
1866+
if prefix.len() > MAX_EXEC_STDIN_BYTES {
18631867
return Err(miette::miette!(
1864-
"stdin payload exceeds {} byte limit; pipe smaller inputs or use `sandbox upload`",
1865-
MAX_STDIN_PAYLOAD
1868+
"piped stdin exceeds the 4 MiB limit; use `sandbox upload` for larger input"
18661869
));
18671870
}
1868-
Ok(buf)
1871+
Ok::<_, miette::Report>(prefix)
18691872
})
18701873
.await
1871-
.into_diagnostic()?? // first ? unwraps JoinError, second ? unwraps Result
1874+
.into_diagnostic()??
18721875
};
18731876

1874-
// Resolve TTY mode: explicit --tty / --no-tty wins, otherwise auto-detect.
1875-
let tty = tty_override
1876-
.unwrap_or_else(|| std::io::stdin().is_terminal() && std::io::stdout().is_terminal());
1877+
let (cols, rows) = if tty {
1878+
local_terminal_size().unwrap_or((80, 24))
1879+
} else {
1880+
(0, 0)
1881+
};
1882+
let mut request = ExecSandboxRequest {
1883+
request_id: String::new(),
1884+
sandbox: name.to_string(),
1885+
workspace_scope: Some(openshell_core::proto::workspace_selector(
1886+
workspace.to_string(),
1887+
)),
1888+
command: command.to_vec(),
1889+
workdir: workdir.unwrap_or_default().to_string(),
1890+
environment: environment.clone(),
1891+
execution_timeout: proto_execution_timeout(timeout_seconds)?,
1892+
stdin: stdin_prefix,
1893+
tty,
1894+
cols,
1895+
rows,
1896+
no_login_shell,
1897+
};
18771898

1878-
if tty && std::io::stdin().is_terminal() {
1879-
return sandbox_exec_interactive_grpc(
1899+
let stdin_for_size_check = std::mem::take(&mut request.stdin);
1900+
let start_request_bytes = request.encoded_len();
1901+
request.stdin = stdin_for_size_check;
1902+
if start_request_bytes > MAX_EXEC_REQUEST_BYTES {
1903+
return Err(miette::miette!(
1904+
"exec command or environment exceeds the gateway's 1 MiB message limit"
1905+
));
1906+
}
1907+
if (tty && stdin_is_terminal) || request.encoded_len() > MAX_EXEC_REQUEST_BYTES {
1908+
return sandbox_exec_streaming_grpc(
18801909
client,
18811910
&sandbox,
18821911
command,
18831912
workdir,
18841913
timeout_seconds,
18851914
environment,
18861915
no_login_shell,
1916+
tty,
1917+
stdin_is_terminal,
1918+
std::mem::take(&mut request.stdin),
18871919
)
18881920
.await;
18891921
}
18901922

1891-
let (cols, rows) = if tty {
1892-
local_terminal_size().unwrap_or_default()
1893-
} else {
1894-
(0, 0)
1895-
};
1896-
18971923
// Make the streaming gRPC call.
18981924
let mut stream = client
1899-
.exec_sandbox(ExecSandboxRequest {
1900-
request_id: String::new(),
1901-
sandbox: name.to_string(),
1902-
workspace_scope: Some(openshell_core::proto::workspace_selector(
1903-
workspace.to_string(),
1904-
)),
1905-
command: command.to_vec(),
1906-
workdir: workdir.unwrap_or_default().to_string(),
1907-
environment: environment.clone(),
1908-
execution_timeout: proto_execution_timeout(timeout_seconds)?,
1909-
stdin: stdin_payload,
1910-
tty,
1911-
cols,
1912-
rows,
1913-
no_login_shell,
1914-
})
1925+
.exec_sandbox(request)
19151926
.await
19161927
.into_diagnostic()?
19171928
.into_inner();
@@ -2279,23 +2290,31 @@ impl Drop for TaskGuard {
22792290
}
22802291
}
22812292

2282-
async fn sandbox_exec_interactive_grpc(
2293+
#[allow(clippy::too_many_arguments)]
2294+
async fn sandbox_exec_streaming_grpc(
22832295
mut client: crate::tls::GrpcClient,
22842296
sandbox: &Sandbox,
22852297
command: &[String],
22862298
workdir: Option<&str>,
22872299
timeout_seconds: u32,
22882300
environment: &HashMap<String, String>,
22892301
no_login_shell: bool,
2302+
tty: bool,
2303+
stdin_is_terminal: bool,
2304+
stdin_prefix: Vec<u8>,
22902305
) -> Result<i32> {
22912306
#[cfg(unix)]
22922307
use openshell_core::proto::ExecSandboxWindowResize;
22932308
use openshell_core::proto::{ExecSandboxInput, exec_sandbox_input};
22942309
use tokio_stream::wrappers::ReceiverStream;
22952310

2296-
let (cols, rows) = local_terminal_size().unwrap_or((80, 24));
2311+
let (cols, rows) = if tty {
2312+
local_terminal_size().unwrap_or((80, 24))
2313+
} else {
2314+
(0, 0)
2315+
};
22972316

2298-
let (input_tx, input_rx) = tokio::sync::mpsc::channel::<ExecSandboxInput>(4096);
2317+
let (input_tx, input_rx) = tokio::sync::mpsc::channel::<ExecSandboxInput>(64);
22992318

23002319
// Send the start message with exec metadata.
23012320
input_tx
@@ -2312,7 +2331,7 @@ async fn sandbox_exec_interactive_grpc(
23122331
no_login_shell,
23132332
execution_timeout: proto_execution_timeout(timeout_seconds)?,
23142333
stdin: Vec::new(),
2315-
tty: true,
2334+
tty,
23162335
cols,
23172336
rows,
23182337
})),
@@ -2326,40 +2345,62 @@ async fn sandbox_exec_interactive_grpc(
23262345
.into_diagnostic()?
23272346
.into_inner();
23282347

2329-
// Enable raw mode so keystrokes are forwarded immediately.
2330-
crossterm::terminal::enable_raw_mode().into_diagnostic()?;
2331-
let raw_guard = RawModeGuard;
2348+
// Raw mode is only appropriate for an interactive terminal, not a pipe.
2349+
let raw_guard = if tty && stdin_is_terminal {
2350+
crossterm::terminal::enable_raw_mode().into_diagnostic()?;
2351+
Some(RawModeGuard)
2352+
} else {
2353+
None
2354+
};
23322355

23332356
// Stdin reader on a detached OS thread. Using std::thread (not
23342357
// spawn_blocking) so the tokio runtime shutdown doesn't wait for a
23352358
// thread blocked on stdin.read(). The thread exits when the channel
23362359
// closes (blocking_send returns Err) or stdin hits EOF.
23372360
let stdin_tx = input_tx.clone();
2361+
let (stdin_result_tx, mut stdin_result_rx) = tokio::sync::oneshot::channel();
23382362
std::thread::spawn(move || {
23392363
let mut stdin = std::io::stdin().lock();
23402364
let mut buf = [0u8; 4096];
2341-
loop {
2342-
match stdin.read(&mut buf) {
2343-
Ok(0) | Err(_) => break,
2344-
Ok(n) => {
2345-
if stdin_tx
2346-
.blocking_send(ExecSandboxInput {
2347-
payload: Some(exec_sandbox_input::Payload::Stdin(buf[..n].to_vec())),
2348-
})
2349-
.is_err()
2350-
{
2351-
break;
2365+
let result = (|| {
2366+
for chunk in stdin_prefix.chunks(buf.len()) {
2367+
if stdin_tx
2368+
.blocking_send(ExecSandboxInput {
2369+
payload: Some(exec_sandbox_input::Payload::Stdin(chunk.to_vec())),
2370+
})
2371+
.is_err()
2372+
{
2373+
return Ok(());
2374+
}
2375+
}
2376+
loop {
2377+
match stdin.read(&mut buf) {
2378+
Ok(0) => return Ok(()),
2379+
Err(error) if error.kind() == ErrorKind::Interrupted => {}
2380+
Err(error) => return Err(error),
2381+
Ok(n) => {
2382+
if stdin_tx
2383+
.blocking_send(ExecSandboxInput {
2384+
payload: Some(exec_sandbox_input::Payload::Stdin(
2385+
buf[..n].to_vec(),
2386+
)),
2387+
})
2388+
.is_err()
2389+
{
2390+
return Ok(());
2391+
}
23522392
}
23532393
}
23542394
}
2355-
}
2395+
})();
2396+
let _ = stdin_result_tx.send(result);
23562397
});
23572398

23582399
// SIGWINCH handler: forward terminal resize events.
23592400
#[cfg(unix)]
2360-
let resize_task = {
2401+
let resize_task = if tty && stdin_is_terminal {
23612402
let resize_tx = input_tx.clone();
2362-
tokio::spawn(async move {
2403+
Some(tokio::spawn(async move {
23632404
let mut sig =
23642405
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::window_change())
23652406
.expect("failed to register SIGWINCH handler");
@@ -2375,17 +2416,56 @@ async fn sandbox_exec_interactive_grpc(
23752416
}
23762417
}
23772418
}
2378-
})
2419+
}))
2420+
} else {
2421+
None
23792422
};
23802423
#[cfg(unix)]
2381-
let _resize_guard = TaskGuard(resize_task);
2424+
let _resize_guard = resize_task.map(TaskGuard);
2425+
2426+
// Keep a sender until the reader confirms clean EOF. On a read error,
2427+
// cancel the response stream before the gateway can treat channel EOF as
2428+
// successful completion of a partial command.
2429+
let mut pipe_input_tx = Some(input_tx);
23822430

23832431
let mut exit_code = 0i32;
23842432
let mut exit_seen = false;
23852433
let stdout = std::io::stdout();
23862434
let stderr = std::io::stderr();
23872435

2388-
while let Some(event) = stream.next().await {
2436+
let mut stdin_reader_done = false;
2437+
loop {
2438+
let event = tokio::select! {
2439+
result = &mut stdin_result_rx, if !stdin_reader_done => {
2440+
stdin_reader_done = true;
2441+
match result.into_diagnostic()? {
2442+
Ok(()) => {
2443+
let sender = pipe_input_tx.take().expect("stdin sender is held until EOF");
2444+
drop(sender);
2445+
}
2446+
Err(error) => {
2447+
let sender = pipe_input_tx.take().expect("stdin sender is held until EOF");
2448+
// A clean request EOF would make the gateway execute
2449+
// the truncated input. An invalid frame makes the
2450+
// gateway abort the command instead.
2451+
let abort = ExecSandboxInput { payload: None };
2452+
if tokio::time::timeout(Duration::from_secs(5), sender.send(abort))
2453+
.await
2454+
.is_err()
2455+
{
2456+
// Keep the request body open if a blocked remote
2457+
// stdin prevents delivery of the abort frame.
2458+
std::mem::forget(sender);
2459+
}
2460+
drop(stream);
2461+
return Err(error).into_diagnostic();
2462+
}
2463+
}
2464+
continue;
2465+
}
2466+
event = stream.next() => event,
2467+
};
2468+
let Some(event) = event else { break };
23892469
let event = event.into_diagnostic()?;
23902470
match event.payload {
23912471
Some(exec_sandbox_event::Payload::Stdout(out)) => {
@@ -2407,11 +2487,13 @@ async fn sandbox_exec_interactive_grpc(
24072487
}
24082488
}
24092489

2410-
drop(input_tx);
2411-
24122490
// Drop the raw mode guard to restore the terminal before returning.
24132491
drop(raw_guard);
24142492

2493+
if !stdin_reader_done && let Ok(result) = stdin_result_rx.try_recv() {
2494+
result.into_diagnostic()?;
2495+
}
2496+
24152497
// A stream that closes without an Exit event means we never observed the
24162498
// command's outcome. Treat it as a relay failure rather than reporting a
24172499
// successful (0) exit.

‎docs/sandboxes/manage-sandboxes.mdx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,13 @@ Pipe stdin into the command:
357357
echo "hello" | openshell sandbox exec -n my-sandbox -- cat
358358
```
359359
360+
The CLI sends small piped input in one request for compatibility with older
361+
gateways. It streams larger input in bounded frames, including for commands
362+
without a TTY, so input is not limited by the gateway's per-message request
363+
size. Piped input is limited to 4 MiB; use `sandbox upload` for larger files.
364+
The CLI closes remote stdin when the pipe reaches EOF and continues
365+
reading command output until the command finishes.
366+
360367
The command's exit code is propagated to the CLI, so `exec` works in scripts that check return codes.
361368
Large stdout and stderr streams are delivered before a successful exit. A slow
362369
reader backpressures the command. If output delivery fails, `exec` returns a

0 commit comments

Comments
 (0)