Skip to content

Commit 0f09a11

Browse files
agentscope-ai-javazhansheng.lzs
andcommitted
fix(acli): give run_command a real timeout and kill the whole tree (#201)
* sync(acli): give run_command a real timeout and kill the whole tree The 30s default killed ordinary test and build commands, and nothing told the model the limit existed -- so it burned turns rediscovering it by experiment (`sleep 25` fits, `sleep 150` does not). Raise the default to 600s, allow ACLI_COMMAND_TIMEOUT to move it, advertise the `timeout` argument in the tool description, and name it in the timeout message. On expiry only the direct child died. proc.kill() reaches the shell, not the pipeline it started, so the work kept running with its stdout pipe open and asyncio later logged "Exception ignored in BaseSubprocessTransport.__del__ ... Event loop is closed". Start every child in its own process group (start_new_session / CREATE_NEW_PROCESS_GROUP), kill the group, then reap it. Ctrl-C takes the same path. /info printed config.timeout as "Timeout", but that field is the LLM request timeout (providers/profile.py), so the one place a user could look reported a limit nothing enforced. Split it into two labelled lines. * test(acli): pin the command timeout and the process-group kill in the vendored tree Orphan coverage is marker-based rather than pid-based: a SIGKILLed child nobody reaps is a zombie, and os.kill(pid, 0) still succeeds on a zombie, so the pid alone says nothing about whether the tree died. Each fix was falsified by reverting it and confirming exactly the intended tests fail. Two results worth recording: reverting 600 -> 30 fails only test_default_covers_a_real_test_run, because the other timeout assertions are self-referential to DEFAULT_TIMEOUT; and dropping the `proc is None` guard in _kill_tree fails nothing, since the AttributeError from None.wait() is swallowed by the broad except. The guard stays so the intent does not depend on an exception being swallowed, but it is not claimed as covered. --------- Co-authored-by: zhansheng.lzs <zhansheng.lzs@alibaba-inc.com>
1 parent e7ea471 commit 0f09a11

4 files changed

Lines changed: 415 additions & 45 deletions

File tree

‎dashscope/acli/cli/dispatch.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,15 @@ def _handle_slash_command(
200200
)
201201
console.print(f" Loop mode: [cyan]{config.loop_mode}[/cyan]")
202202
console.print(f" Max turns: [cyan]{config.max_turns}[/cyan]")
203-
console.print(f" Timeout: [cyan]{config.timeout}s[/cyan]")
203+
# config.timeout is the LLM request timeout; run_command enforces its
204+
# own, so showing one "Timeout" line reported a limit nothing used.
205+
from dashscope.acli.tools.shell import default_timeout
206+
207+
console.print(f" LLM timeout: [cyan]{config.timeout}s[/cyan]")
208+
console.print(
209+
f" Cmd timeout: [cyan]{default_timeout()}s "
210+
f"(ACLI_COMMAND_TIMEOUT)[/cyan]",
211+
)
204212
console.print(
205213
f" TUI: {'[green]on' if config.tui else '[dim]off'}[/]",
206214
)

‎dashscope/acli/config.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,8 @@ class Config:
253253
# "dangerous" - only DANGEROUS prompts; CONFIRM auto-passes
254254
confirm_mode: str = "dangerous"
255255
max_turns: int = 50
256+
# LLM request timeout (providers/profile.py). run_command has its own,
257+
# separate limit: see acli.tools.shell.default_timeout().
256258
timeout: int = 30
257259
mcp_servers: list[MCPServerConfig] = field(default_factory=list)
258260
memory_enabled: bool = True

‎dashscope/acli/tools/shell.py‎

Lines changed: 134 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,38 @@
88
import re
99
import shlex
1010
import shutil
11+
import signal
12+
import subprocess
1113
import sys
1214

1315
from dashscope.acli.tools.registry import PermissionLevel, tool
1416

1517
MAX_OUTPUT_LENGTH = 10000
1618
MAX_OUTPUT_BYTES = 5 * 1024 * 1024 # 5 MB hard limit before truncation
17-
DEFAULT_TIMEOUT = 30
19+
# Sized for real work: a project's test or build command. The previous 30s
20+
# silently killed both, and since the model could not see the limit it spent
21+
# turns guessing at it (e.g. discovering `sleep 25` works and `sleep 150`
22+
# does not). Callers override per call with the ``timeout`` argument.
23+
DEFAULT_TIMEOUT = 600
24+
TIMEOUT_ENV = "ACLI_COMMAND_TIMEOUT"
1825
IS_WINDOWS = os.name == "nt"
26+
# Windows has no setsid/killpg; a new process group is the closest thing, and
27+
# what lets ``taskkill /T`` reach grandchildren. The flag does not exist on
28+
# POSIX, hence getattr.
29+
_NEW_GROUP_FLAGS = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
30+
31+
32+
def default_timeout() -> int:
33+
"""Seconds a run_command call gets before it is killed.
34+
35+
Resolved per call rather than at import, so exporting
36+
``ACLI_COMMAND_TIMEOUT`` takes effect without a restart.
37+
"""
38+
try:
39+
value = int(os.environ.get(TIMEOUT_ENV, "") or DEFAULT_TIMEOUT)
40+
except ValueError:
41+
return DEFAULT_TIMEOUT
42+
return value if value > 0 else DEFAULT_TIMEOUT
1943

2044

2145
def _resolve_output_encoding() -> str:
@@ -519,11 +543,101 @@ def _is_safe_single(chunk: str) -> bool:
519543
return False
520544

521545

546+
def _signal_tree(proc: asyncio.subprocess.Process | None) -> None:
547+
"""Kill the process group. Synchronous: no awaits, so it is safe to call
548+
from a ``CancelledError`` handler where awaiting is unreliable.
549+
"""
550+
if proc is None or proc.returncode is not None:
551+
return
552+
if IS_WINDOWS:
553+
try:
554+
subprocess.run(
555+
["taskkill", "/F", "/T", "/PID", str(proc.pid)],
556+
stdout=subprocess.DEVNULL,
557+
stderr=subprocess.DEVNULL,
558+
check=False,
559+
timeout=10,
560+
)
561+
except Exception: # pylint: disable=broad-except
562+
pass
563+
return
564+
try:
565+
# proc.pid is the group id: the child was started with
566+
# start_new_session, which is what makes this reach grandchildren.
567+
os.killpg(proc.pid, signal.SIGKILL)
568+
except OSError:
569+
try:
570+
proc.kill()
571+
except Exception: # pylint: disable=broad-except
572+
pass
573+
574+
575+
async def _kill_tree(proc: asyncio.subprocess.Process | None) -> None:
576+
"""Kill the tree, then reap it.
577+
578+
Without the reap the transport stays open until GC, which asyncio reports
579+
as "Exception ignored in BaseSubprocessTransport.__del__ ... Event loop is
580+
closed" when the process exits first.
581+
"""
582+
if proc is None:
583+
# The spawn itself failed, so there is nothing to clean up.
584+
return
585+
_signal_tree(proc)
586+
try:
587+
await proc.wait()
588+
except Exception: # pylint: disable=broad-except
589+
pass
590+
591+
592+
async def _spawn(command: str) -> asyncio.subprocess.Process:
593+
"""Start the command in its own process group.
594+
595+
The group is what lets ``_signal_tree`` reach grandchildren. Without it a
596+
timeout killed only the shell, so ``cmd | slow`` kept running after the
597+
caller was told the command had timed out.
598+
"""
599+
if IS_WINDOWS:
600+
# argv form, never a shell string: interpolating into
601+
# 'powershell -Command "..."' routes it through cmd.exe first and
602+
# any embedded double quote corrupts the command.
603+
return await asyncio.create_subprocess_exec(
604+
*WIN_SHELL,
605+
command,
606+
stdout=asyncio.subprocess.PIPE,
607+
stderr=asyncio.subprocess.PIPE,
608+
cwd=os.getcwd(),
609+
creationflags=_NEW_GROUP_FLAGS,
610+
)
611+
# Optional OS sandbox (opt-in; degrades to normal execution
612+
# when disabled or when no backend is available).
613+
from dashscope.acli import sandbox
614+
615+
sandbox_argv = None
616+
if sandbox.is_enabled():
617+
sandbox_argv = sandbox.build_argv(command, os.getcwd())
618+
if sandbox_argv is not None:
619+
return await asyncio.create_subprocess_exec(
620+
*sandbox_argv,
621+
stdout=asyncio.subprocess.PIPE,
622+
stderr=asyncio.subprocess.PIPE,
623+
cwd=os.getcwd(),
624+
start_new_session=True,
625+
)
626+
return await asyncio.create_subprocess_shell(
627+
command,
628+
stdout=asyncio.subprocess.PIPE,
629+
stderr=asyncio.subprocess.PIPE,
630+
cwd=os.getcwd(),
631+
start_new_session=True,
632+
)
633+
634+
522635
@tool(
523636
name="run_command",
524637
description=(
525638
"Execute a shell command and return its output. "
526-
"Dangerous commands are blocked."
639+
"Dangerous commands are blocked. Pass timeout (seconds) to override "
640+
f"the {DEFAULT_TIMEOUT}s default for long builds or test runs."
527641
),
528642
permission=PermissionLevel.CONFIRM,
529643
)
@@ -555,57 +669,33 @@ async def run_command(command: str, timeout: int | None = None) -> str:
555669
# fall back to the default instead of letting asyncio.wait_for raise
556670
# TypeError on `<= 0`.
557671
try:
558-
timeout = (
559-
int(timeout) if timeout not in (None, "", 0) else DEFAULT_TIMEOUT
560-
)
672+
timeout = int(timeout) if timeout not in (None, "", 0) else 0
561673
except (TypeError, ValueError):
562-
timeout = DEFAULT_TIMEOUT
674+
timeout = 0
675+
if timeout <= 0:
676+
timeout = default_timeout()
563677

678+
proc = None
564679
try:
565-
if IS_WINDOWS:
566-
# argv form, never a shell string: interpolating into
567-
# 'powershell -Command "..."' routes it through cmd.exe first and
568-
# any embedded double quote corrupts the command.
569-
proc = await asyncio.create_subprocess_exec(
570-
*WIN_SHELL,
571-
command,
572-
stdout=asyncio.subprocess.PIPE,
573-
stderr=asyncio.subprocess.PIPE,
574-
cwd=os.getcwd(),
575-
)
576-
else:
577-
# Optional OS sandbox (opt-in; degrades to normal execution
578-
# when disabled or when no backend is available).
579-
from dashscope.acli import sandbox
580-
581-
sandbox_argv = None
582-
if sandbox.is_enabled():
583-
sandbox_argv = sandbox.build_argv(command, os.getcwd())
584-
if sandbox_argv is not None:
585-
proc = await asyncio.create_subprocess_exec(
586-
*sandbox_argv,
587-
stdout=asyncio.subprocess.PIPE,
588-
stderr=asyncio.subprocess.PIPE,
589-
cwd=os.getcwd(),
590-
)
591-
else:
592-
proc = await asyncio.create_subprocess_shell(
593-
command,
594-
stdout=asyncio.subprocess.PIPE,
595-
stderr=asyncio.subprocess.PIPE,
596-
cwd=os.getcwd(),
597-
)
680+
proc = await _spawn(command)
598681
stdout, stderr = await asyncio.wait_for(
599682
proc.communicate(),
600683
timeout=timeout,
601684
)
602685
except asyncio.TimeoutError:
603-
try:
604-
proc.kill()
605-
except Exception:
606-
pass
607-
return f"Error: command timed out ({timeout}s)"
686+
await _kill_tree(proc)
687+
return (
688+
f"Error: command timed out after {timeout}s and was killed. "
689+
"Pass a larger timeout argument (seconds) if it needs longer; "
690+
f"the default is {default_timeout()}s."
691+
)
692+
except asyncio.CancelledError:
693+
# Ctrl-C: kill synchronously, because awaiting inside a cancellation
694+
# handler can be interrupted before the kill lands.
695+
_signal_tree(proc)
696+
raise
608697
except Exception as e:
698+
await _kill_tree(proc)
609699
return f"Error: execution failed - {e}"
610700

611701
# Enforce hard size limits to prevent OOM on huge outputs

0 commit comments

Comments
 (0)