diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 71bcba70..9107c5e2 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -107,9 +107,10 @@ class SessionState(str, Enum): READY_ENDING = "ready_ending" """The state of a Session when it is ready to run only Environment End actions. - The Session has previously experienced an error or cancelation running - one of its actions, and is now only allowed to run Environment End actions to - clean up the Session context.""" + The Session enters it after an action ends FAILED, CANCELED or TIMEOUT, or after + exit_environment is called without keep_session_running. It then only accepts + Environment exits and cleanup, and returns to READY only when the caller calls + Session.clear_ending_only().""" ENDED = "ended" """Terminal state of a Session that has ended and can no longer run any actions.""" @@ -251,6 +252,10 @@ class Session(object): """ _state: SessionState + """The stored lifecycle state. It is never READY_ENDING: ending-only mode is + stored only in _ending_only, and the state property derives READY_ENDING + from READY plus that flag. Read the state through the property. + """ _session_id: str """The application-provided id for this Session. @@ -262,7 +267,10 @@ class Session(object): _ending_only: bool """The Session has previously experienced an error or cancelation of a running - action, and can only run Environment-end actions. + action, or exited an Environment without keep_session_running, and can only run + Environment-end actions. This flag is the only record of ending-only mode: it + persists through RUNNING, and the state property reports READY with it set as + READY_ENDING. """ _environments: dict[ @@ -667,8 +675,21 @@ def files_directory(self) -> Path: @property def state(self) -> SessionState: - """Fetch the current state of this Session.""" - return self._state + """Fetch the current state of this Session. + + READY_ENDING is derived, not stored: it is READY while the Session is + ending-only, so the state and the ending-only flag can't disagree. + """ + # Read _state once. Another thread can store READY between two reads, + # and _ending_only is always set before READY is stored for a failed + # action, so a single read never reports READY for one. The flag is + # read second, so a reader that stays paused between the two reads + # while a whole failing action runs reports READY_ENDING a moment + # early, while _state is still RUNNING; it never reports READY late. + state = self._state + if state == SessionState.READY and self._ending_only: + return SessionState.READY_ENDING + return state @property def action_status(self) -> Optional[ActionStatus]: @@ -727,6 +748,35 @@ def cancel_action( runner.cancel(time_limit=time_limit, mark_action_failed=mark_action_failed) + def clear_ending_only(self) -> None: + """Take the Session out of ending-only mode, moving it from READY_ENDING back + to READY so that it accepts run_task, enter_environment and run_subprocess again. + + Use with care. By default a Session is brittle: once an action ends FAILED, + CANCELED or TIMEOUT, or exit_environment is called without keep_session_running, + the Session only allows exiting the Environments it entered and cleanup. This + method is for callers that deliberately want to keep running work after that. + The caller takes responsibility for the Session's state: a failed or canceled + action can leave its work half done, and the Session neither checks nor undoes + any of it. + + This is the only way from READY_ENDING back to READY; + exit_environment(keep_session_running=True) never does it. The method clears + ending-only mode whatever set it. It runs no action and changes nothing else: + the entered Environments, the last action's status and the environment + variables stay as they were. + + Raises: + RuntimeError: If the Session is not in the READY_ENDING state. + """ + if self.state != SessionState.READY_ENDING: + raise RuntimeError( + "Session must be in the READY_ENDING state to clear ending-only mode." + ) + # The stored state is already READY: clearing the flag is all that takes + # the Session from READY_ENDING to READY. + self._ending_only = False + def _make_env_script_runner( self, *, @@ -1060,9 +1110,14 @@ def exit_environment( by values defined in Environments. Key: Environment variable name Value: Value for the environment variable. - keep_session_running (bool): This overrides the default of requiring only environment exits after - the first exit_environment is called. The caller can set this to True in order to exit - the environments of a step and then run tasks from a different step. + keep_session_running (bool): By default, the first exit_environment makes the Session + ending-only, so it then only accepts environment exits and cleanup. Set this to True + to skip that for this exit, for example to exit the environments of a step and then + run tasks from a different step. An onExit action that ends FAILED, CANCELED or + TIMEOUT still makes the Session ending-only, like any other action. This never + brings an ending-only Session back to READY: once an action has ended FAILED, + CANCELED or TIMEOUT, or an earlier exit made the Session ending-only, it stays in + READY_ENDING until the caller calls clear_ending_only(). resolved_symtab (Optional[SerializedSymbolTable]): See :meth:`enter_environment` for semantics. Pass the same table the environment was entered with so its onExit resolves in @@ -2531,7 +2586,11 @@ def _fail_action_before_start(self, message: str) -> None: self._action_fail_message = message self._action_exit_code = None self._action_state = ActionState.FAILED - self._state = SessionState.READY_ENDING + # Sessions are "brittle": a failed action means we can only exit the Session, + # until the caller calls clear_ending_only(). Set the flag before storing + # READY, so the state property never reports READY for a failed action. + self._ending_only = True + self._state = SessionState.READY if self._callback: action_status = self.action_status # R5-6: a plain check, not `assert` -- `_action_state` is assigned @@ -2590,12 +2649,15 @@ def _action_callback(self, state: ActionState) -> None: if state != ActionState.RUNNING: # Decide which between-action state to enter. - if self._ending_only or self._action_state != ActionState.SUCCESS: + if self._action_state != ActionState.SUCCESS: # Sessions are "brittle". If there's a Task cancel or Failure then - # we can only exit the Session. - self._state = SessionState.READY_ENDING - else: - self._state = SessionState.READY + # we can only exit the Session, even if a later exit_environment + # passes keep_session_running=True, until the caller calls + # clear_ending_only(). + self._ending_only = True + # The stored state is READY either way. The state property reports + # READY_ENDING while _ending_only is set, so the flag is set first. + self._state = SessionState.READY if self._callback and action_status is not None: self._callback(self._session_id, action_status) diff --git a/test/openjd/sessions_v0/test_brittle_session.py b/test/openjd/sessions_v0/test_brittle_session.py new file mode 100644 index 00000000..443569ec --- /dev/null +++ b/test/openjd/sessions_v0/test_brittle_session.py @@ -0,0 +1,717 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Brittle sessions: once an action ends FAILED, CANCELED or TIMEOUT, the +Session only accepts environment exits, cleanup and clear_ending_only, and only +clear_ending_only returns it to READY, whatever a later exit_environment passes +as keep_session_running. +""" + +import time +import uuid +from typing import Optional +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.model import ParameterValue +from openjd.model.v2023_09 import Action as Action_2023_09 +from openjd.model.v2023_09 import ArgString as ArgString_2023_09 +from openjd.model.v2023_09 import CommandString as CommandString_2023_09 +from openjd.model.v2023_09 import Environment as Environment_2023_09 +from openjd.model.v2023_09 import EnvironmentActions as EnvironmentActions_2023_09 +from openjd.model.v2023_09 import EnvironmentScript as EnvironmentScript_2023_09 +from openjd.model.v2023_09 import ( + EnvironmentVariableValueString as EnvironmentVariableValueString_2023_09, +) +from openjd.model.v2023_09 import StepActions as StepActions_2023_09 +from openjd.model.v2023_09 import StepScript as StepScript_2023_09 +from openjd.sessions import ActionState, Session, SessionState + +from .conftest import serial_process + + +def _action(python_exe: str, code: str, timeout: Optional[int] = None) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09(code)], + timeout=timeout, + ) + + +def _step(action: Action_2023_09) -> StepScript_2023_09: + return StepScript_2023_09(actions=StepActions_2023_09(onRun=action)) + + +def _env( + name: str, + on_enter: Optional[Action_2023_09] = None, + on_exit: Optional[Action_2023_09] = None, +) -> Environment_2023_09: + if on_enter is None and on_exit is None: + # An environment needs a script or variables. + return Environment_2023_09( + name=name, + variables={"BRITTLE_TEST": EnvironmentVariableValueString_2023_09("1")}, + ) + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(onEnter=on_enter, onExit=on_exit) + ), + ) + + +def _wait(session: Session) -> None: + deadline = time.monotonic() + 20 + while session.state in (SessionState.RUNNING, SessionState.CANCELING): + assert time.monotonic() < deadline, "action did not finish" + time.sleep(0.05) + + +def _no_params() -> dict[str, ParameterValue]: + return dict[str, ParameterValue]() + + +def _assert_refuses_new_actions(session: Session, python_exe: str) -> None: + with pytest.raises(RuntimeError, match="READY state"): + session.run_task( + step_script=_step(_action(python_exe, "print('again')")), + task_parameter_values=_no_params(), + ) + with pytest.raises(RuntimeError, match="READY state"): + session.enter_environment(environment=_env("New", on_enter=_action(python_exe, "pass"))) + with pytest.raises(RuntimeError, match="READY state"): + session.enter_environment(environment=_env("NoAction")) + with pytest.raises(RuntimeError, match="READY state"): + session.run_subprocess(command=python_exe, args=["-c", "pass"]) + + +def _run_ending_task(session: Session, python_exe: str, ending: ActionState) -> None: + if ending == ActionState.FAILED: + action = _action(python_exe, "import sys; sys.exit(1)") + elif ending == ActionState.CANCELED: + action = _action(python_exe, "import time; time.sleep(30)") + else: + action = _action(python_exe, "import time; time.sleep(30)", timeout=1) + session.run_task(step_script=_step(action), task_parameter_values=_no_params()) + if ending == ActionState.CANCELED: + time.sleep(0.5) + session.cancel_action() + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ending + assert session.state == SessionState.READY_ENDING + + +@serial_process +class TestBrittleSession: + @pytest.mark.parametrize( + "ending", [ActionState.FAILED, ActionState.CANCELED, ActionState.TIMEOUT] + ) + @pytest.mark.parametrize("keep_session_running", [True, False]) + @pytest.mark.parametrize("with_on_exit", [True, False]) + def test_task_ending_then_exit( + self, + python_exe: str, + ending: ActionState, + keep_session_running: bool, + with_on_exit: bool, + ) -> None: + # GIVEN + on_exit = _action(python_exe, "print('exited')") if with_on_exit else None + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment(environment=_env("Env", on_exit=on_exit)) + _wait(session) + assert session.state == SessionState.READY + _run_ending_task(session, python_exe, ending) + _assert_refuses_new_actions(session, python_exe) + + # WHEN + session.exit_environment(identifier=env_id, keep_session_running=keep_session_running) + _wait(session) + + # THEN + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + def test_failed_on_enter_then_exit_keep_running(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment( + environment=_env("Env", on_enter=_action(python_exe, "import sys; sys.exit(3)")) + ) + _wait(session) + assert session.state == SessionState.READY_ENDING + + session.exit_environment(identifier=env_id, keep_session_running=True) + _wait(session) + + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + def test_failed_on_exit_then_exit_outer_keep_running(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + outer = session.enter_environment(environment=_env("Outer")) + _wait(session) + inner = session.enter_environment( + environment=_env("Inner", on_exit=_action(python_exe, "import sys; sys.exit(4)")) + ) + _wait(session) + session.exit_environment(identifier=inner, keep_session_running=True) + _wait(session) + assert session.state == SessionState.READY_ENDING + + session.exit_environment(identifier=outer, keep_session_running=True) + _wait(session) + + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + def test_unresolvable_format_string_then_exit_keep_running(self, python_exe: str) -> None: + # A format string naming an undefined parameter fails the action in the + # runner, before its subprocess starts. + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment(environment=_env("Env")) + _wait(session) + session.run_task( + step_script=_step(_action(python_exe, "print('{{ Task.Param.Missing }}')")), + task_parameter_values=_no_params(), + ) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.FAILED + assert session.state == SessionState.READY_ENDING + + session.exit_environment(identifier=env_id, keep_session_running=True) + _wait(session) + + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + def test_failure_before_start_then_exit_keep_running(self, python_exe: str) -> None: + # A failure while the Session prepares the action, before any runner + # exists, goes through _fail_action_before_start. + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment(environment=_env("Env")) + _wait(session) + with patch.object( + session, "_materialize_path_mapping", side_effect=RuntimeError("boom") + ): + session.run_task( + step_script=_step(_action(python_exe, "print('never runs')")), + task_parameter_values=_no_params(), + ) + assert session.action_status is not None + assert session.action_status.state == ActionState.FAILED + assert session.action_status.fail_message == "boom" + assert session.state == SessionState.READY_ENDING + + session.exit_environment(identifier=env_id, keep_session_running=True) + _wait(session) + + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + def test_failed_subprocess_then_exit_keep_running(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment(environment=_env("Env")) + _wait(session) + session.run_subprocess(command=python_exe, args=["-c", "import sys; sys.exit(5)"]) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.FAILED + assert session.state == SessionState.READY_ENDING + + session.exit_environment(identifier=env_id, keep_session_running=True) + _wait(session) + + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + def test_cancel_marked_failed_then_exit_keep_running(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment(environment=_env("Env")) + _wait(session) + session.run_task( + step_script=_step(_action(python_exe, "import time; time.sleep(30)")), + task_parameter_values=_no_params(), + ) + time.sleep(0.5) + session.cancel_action(mark_action_failed=True) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.FAILED + assert session.state == SessionState.READY_ENDING + + session.exit_environment(identifier=env_id, keep_session_running=True) + _wait(session) + + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + @pytest.mark.parametrize( + "keep_session_running, expected", + [(True, SessionState.READY), (False, SessionState.READY_ENDING)], + ) + def test_successful_actions_keep_running_controls_reopening( + self, python_exe: str, keep_session_running: bool, expected: SessionState + ) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment( + environment=_env("Env", on_exit=_action(python_exe, "print('bye')")) + ) + _wait(session) + session.run_task( + step_script=_step(_action(python_exe, "print('ok')")), + task_parameter_values=_no_params(), + ) + _wait(session) + assert session.state == SessionState.READY + + session.exit_environment(identifier=env_id, keep_session_running=keep_session_running) + _wait(session) + + assert session.state == expected + if expected == SessionState.READY: + session.run_task( + step_script=_step(_action(python_exe, "print('again')")), + task_parameter_values=_no_params(), + ) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + assert session.state == SessionState.READY + else: + _assert_refuses_new_actions(session, python_exe) + + +def _assert_accepts_new_actions(session: Session, python_exe: str) -> None: + """A task runs, and a new environment can be entered and exited. Leaves the + Session READY with the same environments entered as before.""" + entered = session.environments_entered + session.run_task( + step_script=_step(_action(python_exe, "print('again')")), + task_parameter_values=_no_params(), + ) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + assert session.state == SessionState.READY + + new_id = session.enter_environment( + environment=_env("New", on_enter=_action(python_exe, "print('new')")) + ) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + assert session.state == SessionState.READY + assert session.environments_entered == entered + (new_id,) + + session.exit_environment(identifier=new_id, keep_session_running=True) + _wait(session) + assert session.state == SessionState.READY + assert session.environments_entered == entered + + session.run_subprocess(command=python_exe, args=["-c", "pass"]) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + assert session.state == SessionState.READY + + +def _var_env( + name: str, value: str, on_exit: Optional[Action_2023_09] = None +) -> Environment_2023_09: + """An Environment that defines CLEAR_TEST_VAR=value, with an optional onExit.""" + return Environment_2023_09( + name=name, + variables={"CLEAR_TEST_VAR": EnvironmentVariableValueString_2023_09(value)}, + script=( + EnvironmentScript_2023_09(actions=EnvironmentActions_2023_09(onExit=on_exit)) + if on_exit is not None + else None + ), + ) + + +def _assert_task_sees_var(session: Session, python_exe: str, value: str) -> None: + """A new task sees CLEAR_TEST_VAR=value, so the variables of the Environments + still entered reach new actions.""" + session.run_task( + step_script=_step( + _action( + python_exe, + f"import os, sys; sys.exit(os.environ.get('CLEAR_TEST_VAR') != {value!r})", + ) + ), + task_parameter_values=_no_params(), + ) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS, f"CLEAR_TEST_VAR must be {value}" + assert session.state == SessionState.READY + + +def _clear_and_check(session: Session) -> None: + """clear_ending_only() moves a READY_ENDING Session to READY without touching + the entered environments, the last action's status or the environment variables.""" + assert session.state == SessionState.READY_ENDING + entered = session.environments_entered + status = session.action_status + env_vars = session._evaluate_current_session_env_vars() + session.clear_ending_only() + assert session.state == SessionState.READY + assert session.environments_entered == entered + assert session.action_status == status + assert session._evaluate_current_session_env_vars() == env_vars + + +def _assert_clear_refused(session: Session) -> None: + state = session.state + entered = session.environments_entered + with pytest.raises( + RuntimeError, + match=r"^Session must be in the READY_ENDING state to clear ending-only mode\.$", + ): + session.clear_ending_only() + assert session.state == state + assert session.environments_entered == entered + + +@serial_process +class TestClearEndingOnly: + @pytest.mark.parametrize( + "ending", [ActionState.FAILED, ActionState.CANCELED, ActionState.TIMEOUT] + ) + @pytest.mark.parametrize("exit_inner_first", [True, False]) + def test_clear_after_task_ending( + self, python_exe: str, ending: ActionState, exit_inner_first: bool + ) -> None: + # GIVEN + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + outer = session.enter_environment( + environment=_var_env( + "Outer", "outer", on_exit=_action(python_exe, "print('outer-exited')") + ) + ) + _wait(session) + inner = session.enter_environment(environment=_var_env("Inner", "inner")) + _wait(session) + _run_ending_task(session, python_exe, ending) + if exit_inner_first: + # keep_session_running=True alone leaves the Session ending-only. + session.exit_environment(identifier=inner, keep_session_running=True) + _wait(session) + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + # WHEN + _clear_and_check(session) + + # THEN + expected = (outer,) if exit_inner_first else (outer, inner) + assert session.environments_entered == expected + # The variables of exactly the Environments still entered reach new + # actions: the exited inner one's override is gone, a kept one stays. + _assert_task_sees_var(session, python_exe, "outer" if exit_inner_first else "inner") + _assert_accepts_new_actions(session, python_exe) + + # An exited environment stays exited, and the rest exit in LIFO order. + if exit_inner_first: + with pytest.raises(RuntimeError, match="Cannot exit unknown Environment"): + session.exit_environment(identifier=inner, keep_session_running=True) + else: + session.exit_environment(identifier=inner, keep_session_running=True) + _wait(session) + session.exit_environment(identifier=outer) + _wait(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + assert session.state == SessionState.READY_ENDING + assert session.environments_entered == () + + @pytest.mark.parametrize("with_on_exit", [True, False]) + def test_clear_after_default_exit(self, python_exe: str, with_on_exit: bool) -> None: + # GIVEN + on_exit = _action(python_exe, "print('bye')") if with_on_exit else None + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + outer = session.enter_environment(environment=_var_env("Outer", "outer")) + _wait(session) + inner = session.enter_environment( + environment=_var_env("Inner", "inner", on_exit=on_exit) + ) + _wait(session) + session.exit_environment(identifier=inner) + _wait(session) + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe) + + # WHEN + _clear_and_check(session) + + # THEN + assert session.environments_entered == (outer,) + _assert_task_sees_var(session, python_exe, "outer") + _assert_accepts_new_actions(session, python_exe) + + def test_clear_keeps_session_env_vars(self, python_exe: str) -> None: + # Variables given to the Session constructor survive the clear. + with Session( + session_id=uuid.uuid4().hex, + job_parameter_values=_no_params(), + os_env_vars={"CLEAR_TEST_VAR": "ctor"}, + ) as session: + _run_ending_task(session, python_exe, ActionState.FAILED) + _clear_and_check(session) + _assert_task_sees_var(session, python_exe, "ctor") + + def test_clear_keeps_action_status(self, python_exe: str) -> None: + # The failing task reports progress, a status message and a fail + # message first, so the before/after comparison in _clear_and_check + # isn't between two Nones. + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + session.run_task( + step_script=_step( + _action( + python_exe, + "import sys; print('openjd_progress: 50'); " + "print('openjd_status: halfway'); " + "print('openjd_fail: clear-test failure'); sys.stdout.flush(); sys.exit(1)", + ) + ), + task_parameter_values=_no_params(), + ) + _wait(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.progress == 50.0 + assert status.status_message == "halfway" + assert status.fail_message is not None + assert status.exit_code is not None + assert session.state == SessionState.READY_ENDING + + _clear_and_check(session) + _assert_accepts_new_actions(session, python_exe) + + def test_clear_after_failure_before_start(self, python_exe: str) -> None: + # Ending-only set by _fail_action_before_start clears the same way. + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment(environment=_var_env("Env", "env")) + _wait(session) + with patch.object( + session, "_materialize_path_mapping", side_effect=RuntimeError("boom") + ): + session.run_task( + step_script=_step(_action(python_exe, "print('never runs')")), + task_parameter_values=_no_params(), + ) + assert session.action_status is not None + assert session.action_status.fail_message == "boom" + + _clear_and_check(session) + assert session.environments_entered == (env_id,) + _assert_task_sees_var(session, python_exe, "env") + _assert_accepts_new_actions(session, python_exe) + + def test_clear_then_fail_again(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + env_id = session.enter_environment(environment=_env("Env")) + _wait(session) + _run_ending_task(session, python_exe, ActionState.FAILED) + _clear_and_check(session) + + _run_ending_task(session, python_exe, ActionState.FAILED) + _assert_refuses_new_actions(session, python_exe) + session.exit_environment(identifier=env_id, keep_session_running=True) + _wait(session) + assert session.state == SessionState.READY_ENDING + + _clear_and_check(session) + _assert_accepts_new_actions(session, python_exe) + + def test_refused_in_ready(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + _assert_clear_refused(session) + + session.enter_environment(environment=_env("Env")) + _wait(session) + session.run_task( + step_script=_step(_action(python_exe, "print('ok')")), + task_parameter_values=_no_params(), + ) + _wait(session) + assert session.state == SessionState.READY + _assert_clear_refused(session) + _assert_accepts_new_actions(session, python_exe) + + def test_refused_while_running(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + session.run_task( + step_script=_step(_action(python_exe, "import time; time.sleep(30)")), + task_parameter_values=_no_params(), + ) + assert session.state == SessionState.RUNNING + _assert_clear_refused(session) + session.cancel_action() + _wait(session) + assert session.state == SessionState.READY_ENDING + + @pytest.mark.parametrize("forced", [SessionState.RUNNING, SessionState.CANCELING]) + def test_refusal_keeps_ending_only(self, python_exe: str, forced: SessionState) -> None: + # A refused call must not clear ending-only mode. The state is forced + # because a failed Session can't start a real action. + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + _run_ending_task(session, python_exe, ActionState.FAILED) + + session._state = forced + _assert_clear_refused(session) + + # Still ending-only: forced back to READY, it reports READY_ENDING. + session._state = SessionState.READY + assert session.state == SessionState.READY_ENDING + + @pytest.mark.parametrize("make_ending_only", [True, False]) + def test_refused_in_ended(self, python_exe: str, make_ending_only: bool) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + if make_ending_only: + _run_ending_task(session, python_exe, ActionState.FAILED) + session.cleanup() + assert session.state == SessionState.ENDED + _assert_clear_refused(session) + + +@serial_process +class TestEndingOnlyStoredInFlag: + """Ending-only mode is stored only in Session._ending_only. The state + property derives READY_ENDING from a stored READY plus the flag, so the + stored state is never READY_ENDING.""" + + @pytest.mark.parametrize( + "stored", + [SessionState.READY, SessionState.RUNNING, SessionState.CANCELING, SessionState.ENDED], + ) + @pytest.mark.parametrize("ending_only", [True, False]) + def test_state_derives_ready_ending(self, stored: SessionState, ending_only: bool) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + session._state = stored + session._ending_only = ending_only + expected = ( + SessionState.READY_ENDING + if stored == SessionState.READY and ending_only + else stored + ) + assert session.state == expected + + def test_failed_action_stores_ready(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + _run_ending_task(session, python_exe, ActionState.FAILED) + assert session._state == SessionState.READY + assert session._ending_only + + # The error names the derived state. + with pytest.raises(RuntimeError, match="Current state: ready_ending$"): + session.run_subprocess(command=python_exe, args=["-c", "pass"]) + + session.clear_ending_only() + assert session._state == SessionState.READY + assert not session._ending_only + assert session.state == SessionState.READY + + def test_failure_before_start_stores_ready(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + session._fail_action_before_start("setup failed") + assert session._state == SessionState.READY + assert session._ending_only + assert session.state == SessionState.READY_ENDING + + def test_default_exit_stores_ready(self, python_exe: str) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + identifier = session.enter_environment( + environment=_env("Env", on_exit=_action(python_exe, "pass")) + ) + _wait(session) + session.exit_environment(identifier=identifier) + _wait(session) + assert session._state == SessionState.READY + assert session._ending_only + assert session.state == SessionState.READY_ENDING + + def test_poller_never_sees_ready_for_a_failed_action(self) -> None: + # The action callback runs on another thread. Run it at the worst + # moment for a reader of the state property: right after the + # property's read of _state. The property must report the state + # before or after the callback, never READY for the failed action. + class _CallbackDuringRead(Session): + armed = False + + def __getattribute__(self, name: str): + value = object.__getattribute__(self, name) + if name == "_state" and object.__getattribute__(self, "armed"): + object.__setattr__(self, "armed", False) + object.__getattribute__(self, "_action_callback")(ActionState.FAILED) + return value + + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + session._runner = MagicMock(exit_code=1) + session._action_state = ActionState.RUNNING + session._state = SessionState.RUNNING + session.__class__ = _CallbackDuringRead + session.armed = True # type: ignore[attr-defined] + + assert session.state == SessionState.RUNNING + assert session.state == SessionState.READY_ENDING + session._runner = None + session.__class__ = Session + + def test_flag_set_before_ready_is_stored(self, python_exe: str) -> None: + # A reader on another thread can't see READY for a failed action + # only if _ending_only is set before READY is stored. + writes: list[tuple[str, object]] = [] + + class _RecordWrites(Session): + def __setattr__(self, name: str, value: object) -> None: + if name in ("_state", "_ending_only"): + writes.append((name, value)) + super().__setattr__(name, value) + + # A failed action, through _action_callback. + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + session.__class__ = _RecordWrites + _run_ending_task(session, python_exe, ActionState.FAILED) + session.__class__ = Session + settle = writes.index( + ("_state", SessionState.READY), writes.index(("_state", SessionState.RUNNING)) + ) + assert ("_ending_only", True) in writes[:settle] + + # A failure before the action starts, through _fail_action_before_start. + writes.clear() + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + session.__class__ = _RecordWrites + session._fail_action_before_start("setup failed") + session.__class__ = Session + assert writes == [("_ending_only", True), ("_state", SessionState.READY)] + + def test_default_exit_that_raises_leaves_ready_ending(self, python_exe: str) -> None: + # exit_environment without keep_session_running makes the Session + # ending-only before it stores RUNNING. If it raises in between, the + # Session reads READY_ENDING and refuses new work, as the exit asked. + # (Before ending-only mode was stored only in the flag, it read READY + # here and accepted run_task.) Calls with correctly typed arguments + # don't raise there, so the failure is injected. + with Session(session_id=uuid.uuid4().hex, job_parameter_values=_no_params()) as session: + identifier = session.enter_environment(environment=_env("Env")) + _wait(session) + assert session.state == SessionState.READY + + with patch.object( + session, "_evaluate_current_session_env_vars", side_effect=RuntimeError("boom") + ): + with pytest.raises(RuntimeError, match="^boom$"): + session.exit_environment(identifier=identifier) + + assert session._state == SessionState.READY + assert session.state == SessionState.READY_ENDING + _assert_refuses_new_actions(session, python_exe)