Skip to content

Commit c69d79e

Browse files
committed
fix: Refuse a second cancel, and bound every cancel-token wait
A fourth review found: - A second cancel_action while the session was CANCELING went through the cancel handle and returned, ignoring its time limit. The crate's Session::cancel_action refuses it with InvalidState; the binding now does the same. - The unbounded wait for the cancel token could hang for ever when the caller held a lock that a logging handler on the action thread needs, since the action thread logs its banner before installing the token. The wait is now bounded at 20 s from other threads, then raises "Cannot cancel: the action has not started yet". On the action thread itself it raises at once, since the token can't appear while that thread waits. - dispatch turned a lost wake-up from the action thread into success. It now raises SessionError "the action thread ended unexpectedly". Nothing produces that today. The sessions spec now says not to call an action method or cancel_action while holding a lock that a logging handler takes on another thread, since each waits for the action thread, which logs. Fix stale comments about the runtime-build failure, and keep the tests' waits under pytest's 30 s limit. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com>
1 parent 16b50ac commit c69d79e

4 files changed

Lines changed: 129 additions & 49 deletions

File tree

‎rust-bindings/src/sessions/session.rs‎

Lines changed: 52 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,9 @@ fn lock_recover<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
145145

146146
/// Build a terminal FAILED `ActionStatus` carrying `message`.
147147
///
148-
/// Used when an action thread can't even start (e.g. the Tokio runtime
149-
/// fails to build) so the Python poller observes a terminal state and
150-
/// stops waiting on RUNNING rather than spinning forever.
148+
/// Used when an accepted action panics, or returns an error the crate
149+
/// didn't record, so the Python poller observes a terminal state and stops
150+
/// waiting on RUNNING rather than spinning forever.
151151
fn failed_action_status(message: &str) -> ActionStatus {
152152
ActionStatus {
153153
state: openjd_sessions::action::ActionState::Failed,
@@ -254,11 +254,11 @@ impl Drop for ActionThreadGuard {
254254
/// poison shared state silently and tear down the interpreter's view
255255
/// of the session.
256256
///
257-
/// A runtime-build failure is surfaced as a terminal FAILED
258-
/// `ActionStatus` so the Python-side poller terminates cleanly. A panic
259-
/// inside the action is likewise converted into a terminal FAILED
260-
/// `ActionStatus` (rather than copying the possibly-still-Running real
261-
/// session state), so the poller never hangs on a panicked action.
257+
/// A runtime-build failure is a refusal: the session goes back unchanged
258+
/// and the caller raises. A panic inside the action is converted into a
259+
/// terminal FAILED `ActionStatus` (rather than copying the
260+
/// possibly-still-Running real session state), so the poller never hangs
261+
/// on a panicked action.
262262
///
263263
/// If the calling Python thread is still waiting (the action never
264264
/// reported `Running`), it is told the outcome after the session is back in
@@ -335,8 +335,7 @@ fn run_action<F>(
335335
// usable for future calls, but the real `session.state()` may
336336
// still read Running with no terminal `ActionStatus` — which
337337
// would leave the Python poller waiting forever. Force a
338-
// terminal FAILED status and drop out of RUNNING, mirroring the
339-
// runtime-build-failure branch above.
338+
// terminal FAILED status and drop out of RUNNING.
340339
{
341340
let mut snap = lock_recover(&snapshot);
342341
snap.state = SessionState::Ready;
@@ -505,11 +504,15 @@ impl PySession {
505504
accepted: false,
506505
error: Some(e),
507506
}) => Err(session_err_to_py(e)),
507+
// `Err`: the action thread ended without saying, which
508+
// `ActionThreadGuard` turns into a wake-up. Nothing does that
509+
// today; if it did, the session may not be back in its slot.
510+
Err(_) => Err(session_err_to_py(SessionError::Runtime(
511+
"the action thread ended unexpectedly".into(),
512+
))),
508513
// Started, or an accepted call that returned (its status says
509-
// how). `Err` means the action thread ended without saying, which
510-
// `ActionThreadGuard` turns into a wake-up; nothing does that
511-
// today.
512-
_ => Ok(()),
514+
// how).
515+
Ok(_) => Ok(()),
513516
}
514517
}
515518
}
@@ -898,11 +901,13 @@ impl PySession {
898901
/// Cancel the running action.
899902
///
900903
/// Raises `SessionError` with the crate's message if no action is
901-
/// running, and `ValueError` if `time_limit` is negative, NaN or
902-
/// infinite. Called from the action's own thread (from a logging
903-
/// handler) before the action has started its subprocess, it waits up
904-
/// to 2 s and then raises `SessionError("Cannot cancel: the action has
905-
/// not started yet")`.
904+
/// running or one is already being canceled, and `ValueError` if
905+
/// `time_limit` is negative, NaN or infinite. Raises
906+
/// `SessionError("Cannot cancel: the action has not started yet")` if
907+
/// the action hasn't installed its cancel token: at once when called
908+
/// from the action's own thread (a logging handler), and after 20 s
909+
/// otherwise, which happens only when the action thread is blocked on a
910+
/// lock the caller holds, such as a logging handler's.
906911
#[pyo3(signature = (time_limit=None, mark_action_failed=None))]
907912
fn cancel_action(
908913
&self,
@@ -920,17 +925,23 @@ impl PySession {
920925
// The crate reports an action `Running` just before it logs the
921926
// action's banner and installs its cancel token, so a cancel straight
922927
// after the call that started it can find no token yet. Logging needs
923-
// the GIL, so the gap is as long as the action thread waits for it,
924-
// which under load can be seconds. Keep trying, with the GIL
925-
// released, until the token is there or the action thread has given
926-
// the session back; both happen without help from this thread. On
927-
// the action thread itself (a logging handler that cancels, say),
928-
// neither can happen while this call waits, so give up after
929-
// `CANCEL_WAIT` there.
930-
const CANCEL_WAIT: std::time::Duration = std::time::Duration::from_secs(2);
928+
// the GIL and runs every Python handler, so the gap is as long as the
929+
// action thread waits for those, which under load can be seconds.
930+
// Keep trying, with the GIL released, until the token is there or the
931+
// action thread has given the session back. Give up after
932+
// `CANCEL_WAIT`, which is reached only when the action thread is
933+
// stuck on something this thread holds, such as a logging handler's
934+
// lock. On the action thread itself (a logging handler that cancels)
935+
// the token can't appear while this call runs, so don't wait at all.
936+
const CANCEL_WAIT: std::time::Duration = std::time::Duration::from_secs(20);
931937
let on_action_thread =
932938
*lock_recover(&self.action_thread) == Some(std::thread::current().id());
933-
let deadline = on_action_thread.then(|| std::time::Instant::now() + CANCEL_WAIT);
939+
let deadline = std::time::Instant::now()
940+
+ if on_action_thread {
941+
std::time::Duration::ZERO
942+
} else {
943+
CANCEL_WAIT
944+
};
934945
py.detach(move || loop {
935946
{
936947
let mut guard = lock_recover(&session_arc);
@@ -952,12 +963,24 @@ impl PySession {
952963
// write its final snapshot before the `Canceling` below, so
953964
// the snapshot can't be left reading `Canceling` after the
954965
// action has ended.
966+
// A second cancel while one is in progress: the crate's
967+
// `Session::cancel_action` refuses it, so do the same rather
968+
// than deliver it through the handle.
969+
{
970+
let snap = lock_recover(&snapshot);
971+
if snap.state == SessionState::Canceling {
972+
return Err(session_err_to_py(SessionError::InvalidState {
973+
expected: vec![SessionState::Running],
974+
current: SessionState::Canceling,
975+
}));
976+
}
977+
}
955978
if handle.cancel(duration, mark_failed) {
956979
lock_recover(&snapshot).state = SessionState::Canceling;
957980
return Ok(());
958981
}
959982
}
960-
if deadline.is_some_and(|d| std::time::Instant::now() >= d) {
983+
if std::time::Instant::now() >= deadline {
961984
return Err(session_err_to_py(SessionError::Runtime(
962985
"Cannot cancel: the action has not started yet".into(),
963986
)));

‎specs/python-sessions-interface.md‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -223,15 +223,25 @@ Once the crate accepts a call, the method returns when its action is
223223
`action_status` describes that action. A `cancel_action` made as soon as
224224
it returns reaches the action, unless the action has already ended, as
225225
one that fails before its subprocess starts can: then it raises the
226-
crate's `InvalidState`. A zero `time_limit` terminates at once. Called
227-
from the action's own thread (a logging handler) before the action has
228-
started, `cancel_action` waits up to 2 s and then raises
229-
`SessionError("Cannot cancel: the action has not started yet")`, the
230-
binding's own message. Every error after acceptance, such as a parameter
231-
value that can't be coerced, is reported on `action_status` and doesn't
232-
raise, even when it arrives before the call returns. The crate documents
233-
those as failing the action rather than refusing the call, and the
234-
pure-Python reference reports them the same way.
226+
crate's `InvalidState`. A zero `time_limit` terminates at once. A second
227+
`cancel_action` while the session is `CANCELING` raises the crate's
228+
`InvalidState`. If the action hasn't installed its cancel token yet,
229+
`cancel_action` raises `SessionError("Cannot cancel: the action has not
230+
started yet")`, the binding's own message: at once when called from the
231+
action's own thread (a logging handler), and after 20 s otherwise. Every
232+
error after acceptance, such as a parameter value that can't be coerced,
233+
is reported on `action_status` and doesn't raise, even when it arrives
234+
before the call returns. The crate documents those as failing the action
235+
rather than refusing the call, and the pure-Python reference reports
236+
them the same way.
237+
238+
The crate runs each action on a background thread, and that thread logs
239+
through Python's `logging` (each record needs the GIL and runs the
240+
handlers). An action method waits for the crate to accept or refuse the
241+
call, and `cancel_action` waits for the action's cancel token, so don't
242+
call either while holding a lock that a logging handler takes on another
243+
thread: the action thread would wait for the lock, and the call for the
244+
action thread.
235245

236246
An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session
237247
ending-only (`READY_ENDING`): `exit_environment` and `cleanup` still

‎src/openjd/_openjd_rs.pyi‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2302,11 +2302,13 @@ class Session:
23022302
Cancel the running action.
23032303
23042304
Raises `SessionError` with the crate's message if no action is
2305-
running, and `ValueError` if `time_limit` is negative, NaN or
2306-
infinite. Called from the action's own thread (from a logging
2307-
handler) before the action has started its subprocess, it waits up
2308-
to 2 s and then raises `SessionError("Cannot cancel: the action has
2309-
not started yet")`.
2305+
running or one is already being canceled, and `ValueError` if
2306+
`time_limit` is negative, NaN or infinite. Raises
2307+
`SessionError("Cannot cancel: the action has not started yet")` if
2308+
the action hasn't installed its cancel token: at once when called
2309+
from the action's own thread (a logging handler), and after 20 s
2310+
otherwise, which happens only when the action thread is blocked on a
2311+
lock the caller holds, such as a logging handler's.
23102312
"""
23112313

23122314
def cleanup(self) -> None:

‎test/openjd/sessions/test_refused_calls.py‎

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,11 @@ def _environment(name: str = "E", *, extensions: Optional[list[str]] = None, **a
6464

6565

6666
def _settle(session: Session) -> None:
67-
deadline = time.monotonic() + 30
67+
deadline = time.monotonic() + 10
6868
busy = (SessionState.RUNNING, SessionState.CANCELING)
6969
while session.state in busy and time.monotonic() < deadline:
7070
time.sleep(0.02)
71-
assert session.state not in busy, "action did not finish within 30s"
71+
assert session.state not in busy, "action did not finish within 10s"
7272

7373

7474
def _status(session: Session) -> Optional[tuple]:
@@ -348,6 +348,51 @@ def test_with_no_action_running(self, session: Session) -> None:
348348
_settle(session)
349349
_assert_refused(session, session.cancel_action, message)
350350

351+
def test_a_second_cancel_while_canceling(self, session: Session, tmp_path: Path) -> None:
352+
"""The crate refuses a cancel while one is in progress. Before, the
353+
binding delivered it through the cancel handle and returned, ignoring
354+
its time limit."""
355+
marker = tmp_path / "started"
356+
ignore_notify = (
357+
"import signal, time\n"
358+
"for name in ('SIGTERM', 'SIGBREAK'):\n"
359+
" if hasattr(signal, name):\n"
360+
" signal.signal(getattr(signal, name), signal.SIG_IGN)\n"
361+
f"open({str(marker)!r}, 'w').close()\n"
362+
"time.sleep(30)"
363+
)
364+
script = deserialize_step(
365+
{
366+
"name": "S",
367+
"script": {
368+
"actions": {
369+
"onRun": {
370+
**_python(ignore_notify),
371+
"cancelation": {
372+
"mode": "NOTIFY_THEN_TERMINATE",
373+
"notifyPeriodInSeconds": 3,
374+
},
375+
}
376+
}
377+
},
378+
}
379+
).script
380+
session.run_task(step_script=script)
381+
deadline = time.monotonic() + 10
382+
while not marker.exists() and time.monotonic() < deadline:
383+
time.sleep(0.02)
384+
assert marker.exists()
385+
session.cancel_action()
386+
assert session.state == SessionState.CANCELING
387+
_assert_refused(
388+
session,
389+
lambda: session.cancel_action(time_limit=0.0),
390+
"Session must be in RUNNING state, current: CANCELING",
391+
)
392+
_settle(session)
393+
status = session.action_status
394+
assert status is not None and status.state == ActionState.CANCELED
395+
351396
def test_a_time_limit_with_no_duration(self, session: Session) -> None:
352397
_assert_refused(
353398
session,
@@ -382,7 +427,7 @@ def test_straight_after_the_call_that_started_the_action(
382427
began = time.monotonic()
383428
session.cancel_action()
384429
_settle(session)
385-
assert time.monotonic() - began < 20
430+
assert time.monotonic() - began < 4
386431
status = session.action_status
387432
assert status is not None and status.state == ActionState.CANCELED
388433
assert session.state == SessionState.READY_ENDING
@@ -498,9 +543,9 @@ def emit(self, record):
498543
def test_cancel_from_the_action_thread_before_it_starts(tmp_path: Path) -> None:
499544
"""The crate logs the "Running Task" banner on the action thread between
500545
reporting the action ``Running`` and installing its cancel token. A
501-
logging handler that cancels there can't wait for the token, so after a
502-
bounded wait it raises instead of hanging. Runs in a fresh interpreter,
503-
as above."""
546+
logging handler that cancels there can't wait for the token, which can't
547+
appear while it runs, so it raises at once instead of hanging. Runs in a
548+
fresh interpreter, as above."""
504549
import subprocess
505550

506551
result = subprocess.run(

0 commit comments

Comments
 (0)