From f2908ebb98ea28560439a46c6cac7a2a3ceb0e80 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:46:22 +0000 Subject: [PATCH 01/21] fix!: Bump the bindings to openjd-expr 0.14, openjd-model 0.15 and openjd-sessions 0.9 Move rust-bindings to openjd-expr 0.14.0, openjd-model 0.15.0 and openjd-sessions 0.9.0. Cargo.lock changes only those three crates and what they pull in: serde-saphyr 1.2.0 -> 2.1.1 and granit-parser 1.2.0 -> 2.1.0. The new crates needed these binding changes: - INT and CHUNK[INT] range list elements are now IntOrFormatString (openjd-rs#486). The template-side range getter exposes each element as an int or a FormatString, as FLOAT elements already were and as the pure-Python reference does. - Counts are u64 on every target (openjd-rs#482). The binding stores and passes them as u64, and __len__ converts to usize, raising OverflowError where it doesn't fit. - Path mapping rules apply through a PathMapper (openjd-rs#472). HostContext.with_rules builds one and raises ValueError for an invalid rule, PathMappingRule.apply builds a one-rule mapper, and Session.extend_path_mapping_rules raises SessionError instead of dropping the new Result. Tests changed for behavior the crates changed on purpose: - A list comprehension over an unresolved value has element type T | nulltype, since a run may keep nothing (openjd-rs#462). - A NONCONTIGUOUS chunk renders in the canonical form of Template Schemas 3.4.1.1.1, so [1, 2, 4, 6] is "1,2-6:2" (openjd-rs#469). The pure-Python reference still writes "1,2,4,6"; a strict xfail in model_v1/test_known_gaps.py records that. - A FLOAT range list element's format string is checked at decode time (openjd-rs#486), as the pure-Python reference already did. New tests pin the other visible changes: INT and CHUNK[INT] format string elements, relative path mapping sources rejected and the rule with the most names winning in any order (openjd-rs#472), empty PATH join operands (openjd-rs#516), and mixed-case tRUE and nULL read as strings (openjd-rs#487). No strict xfail tracking an openjd-rs gap XPASSes, so none are removed. The specs and the type stub's docstrings describe the new behavior. No public signature changed. BREAKING CHANGE: PathMappingRule.apply and HostContext.with_rules raise ValueError, and Session raises SessionError, for a rule whose POSIX or WINDOWS source_path is not absolute. When several rules match, the one whose source path has the most names applies, not the first in the list. IntTaskParameterDefinition.range and ChunkIntTaskParameterDefinition.range list elements can be FormatString as well as int. A NONCONTIGUOUS chunk value is written in canonical form. An unresolved list comprehension's type is list[T | nulltype]. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- Cargo.lock | 20 ++--- rust-bindings/Cargo.toml | 6 +- rust-bindings/src/expr/expr_value.rs | 2 +- rust-bindings/src/expr/path_mapping.rs | 24 ++++-- rust-bindings/src/expr/profile.rs | 24 ++++-- rust-bindings/src/expr/range_expr.rs | 16 ++-- rust-bindings/src/lib.rs | 12 +++ rust-bindings/src/model/step_param_space.rs | 24 +++--- .../src/model/step_param_space_def.rs | 8 +- rust-bindings/src/model/task_parameter.rs | 8 +- rust-bindings/src/sessions/session.rs | 14 ++-- specs/python-expr-interface.md | 15 ++++ specs/python-model-interface.md | 20 +++-- specs/python-sessions-interface.md | 8 ++ src/openjd/_openjd_rs.pyi | 22 ++++- test/openjd/expr/test_path_mapping.py | 81 +++++++++++++++++++ test/openjd/expr/test_paths.py | 20 ++++- test/openjd/expr/test_unresolved_eval.py | 55 +++++++++---- test/openjd/model_v1/test_known_gaps.py | 27 +++++++ test/openjd/model_v1/test_parse.py | 19 +++++ .../model_v1/test_step_param_space_def.py | 78 +++++++++++++++++- .../model_v1/test_step_param_space_iter.py | 29 ++++--- .../sessions/test_path_mapping_rules.py | 62 ++++++++++++++ 23 files changed, 491 insertions(+), 103 deletions(-) create mode 100644 test/openjd/sessions/test_path_mapping_rules.py diff --git a/Cargo.lock b/Cargo.lock index 6bdf6dbc..14622577 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "granit-parser" -version = "1.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ec0d45986cd51c847c75c5b69a00852c4fc84d0e5e79f041173f73437d0cdf" +checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" dependencies = [ "arraydeque", "smallvec", @@ -791,9 +791,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openjd-expr" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "febc1f6e714779847dfbb7c0754c274febc286a668076267612a8afa3db2fcee" +checksum = "e2aa8e58546ad6ee7baf11ad8a547694f0d49863e635e7f1a7ac000e6b79123b" dependencies = [ "regex", "regex-syntax", @@ -808,9 +808,9 @@ dependencies = [ [[package]] name = "openjd-model" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e8a7a5c765e901c5ba5e38b01775bdb9064fd9a3161eae1f241dbd66200bae9" +checksum = "144e06e705f61da6e83b64ebf0c237a46199de28e8deadc7fdd26acb44ccf139" dependencies = [ "indexmap", "openjd-expr", @@ -840,9 +840,9 @@ dependencies = [ [[package]] name = "openjd-sessions" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86b899ad49220c5c57a223d57f8adf421eb427ad6013f8047d07806931eb8ff" +checksum = "e32109f68122a780b053c782be55d365ea7486a025ae3e4dad22e83065808698" dependencies = [ "bitflags", "futures-util", @@ -1378,9 +1378,9 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "1.2.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3afb591f9cdb6223c88ba39269aff895620c7f0716dc42b705b5733d5c7c0823" +checksum = "d43fb82d550b07e6f929a58d7d0935182602f48ddead875f5ee313c41c8f07fe" dependencies = [ "annotate-snippets", "base64", diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index 3306161c..5157a7fc 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -12,9 +12,9 @@ name = "_openjd_rs" crate-type = ["cdylib", "rlib"] [dependencies] -openjd-expr = "0.12.0" -openjd-model = "0.13.0" -openjd-sessions = "0.8.1" +openjd-expr = "0.14.0" +openjd-model = "0.15.0" +openjd-sessions = "0.9.0" tokio = { version = "1", features = ["rt-multi-thread"] } uuid = { version = "1", features = ["v4"] } serde_json = "1" diff --git a/rust-bindings/src/expr/expr_value.rs b/rust-bindings/src/expr/expr_value.rs index 9aa1eb58..0d093135 100644 --- a/rust-bindings/src/expr/expr_value.rs +++ b/rust-bindings/src/expr/expr_value.rs @@ -331,7 +331,7 @@ impl PyExprValue { fn __len__(&self) -> PyResult { match &self.inner { - ExprValue::RangeExpr(r) => Ok(r.len()), + ExprValue::RangeExpr(r) => crate::len_to_usize(r.len()), _ => self.inner.list_len().ok_or_else(|| { pyo3::exceptions::PyTypeError::new_err("ExprValue is not a list or range_expr") }), diff --git a/rust-bindings/src/expr/path_mapping.rs b/rust-bindings/src/expr/path_mapping.rs index 77a8c6b0..98560bd2 100644 --- a/rust-bindings/src/expr/path_mapping.rs +++ b/rust-bindings/src/expr/path_mapping.rs @@ -6,7 +6,7 @@ use pyo3::types::PyDict; #[cfg(feature = "stub-gen")] use pyo3_stub_gen::derive::*; -use openjd_expr::path_mapping::{PathFormat, PathMappingRule}; +use openjd_expr::path_mapping::{PathFormat, PathMapper, PathMappingRule}; use crate::expr::path_format::PyPathFormat; @@ -167,16 +167,28 @@ impl PyPathMappingRule { h.finish() } + /// Apply this rule to `path`. Returns `(True, mapped)` when the rule + /// matches and `(False, path)` when it does not. Raises `ValueError` + /// when the rule itself is invalid, for example a POSIX or WINDOWS + /// `source_path` that is not absolute. #[pyo3(signature = (*, path, output_format=None))] - fn apply(&self, path: &str, output_format: Option) -> (bool, String) { + fn apply(&self, path: &str, output_format: Option) -> PyResult<(bool, String)> { + // openjd-expr applies rules only through a `PathMapper`, so a + // one-rule mapper gives this rule's mapping. `validate` first so the + // error does not name an index in a list the caller never made. + let invalid = |e: openjd_expr::path_mapping::InvalidPathMappingRule| { + pyo3::exceptions::PyValueError::new_err(e.to_string()) + }; + self.inner.validate().map_err(invalid)?; + let mapper = PathMapper::new(vec![self.inner.clone()]).map_err(invalid)?; let result = match output_format { - Some(fmt) => self.inner.apply_with_format(path, fmt.into()), - None => self.inner.apply(path), + Some(fmt) => mapper.apply_with_format(path, fmt.into()), + None => mapper.apply(path), }; - match result { + Ok(match result { Some(mapped) => (true, mapped), None => (false, path.to_string()), - } + }) } fn to_dict(&self) -> std::collections::HashMap { diff --git a/rust-bindings/src/expr/profile.rs b/rust-bindings/src/expr/profile.rs index 55f3897f..96791d9a 100644 --- a/rust-bindings/src/expr/profile.rs +++ b/rust-bindings/src/expr/profile.rs @@ -22,6 +22,7 @@ use pyo3::types::{PyDict, PyType}; #[cfg(feature = "stub-gen")] use pyo3_stub_gen::derive::*; +use openjd_expr::path_mapping::PathMapper; use openjd_expr::profile::{ExprExtension, ExprProfile, ExprRevision, HostContext}; use crate::expr::path_mapping::PyPathMappingRule; @@ -237,13 +238,19 @@ impl PyHostContext { /// host context (`apply_path_mapping` is registered, returns /// the path unchanged). Passing zero rules is *not* the same as /// passing no host context at all. + /// + /// Raises `ValueError` when a rule is invalid, for example a POSIX or + /// WINDOWS `source_path` that is not absolute. The message names the + /// rule's index in `rules`. #[classmethod] - fn with_rules(_cls: &Bound<'_, PyType>, rules: Vec) -> Self { + fn with_rules(_cls: &Bound<'_, PyType>, rules: Vec) -> PyResult { let rust_rules: Vec = rules.into_iter().map(|r| r.inner).collect(); - Self { - inner: HostContext::WithRules(Arc::new(rust_rules)), - } + let mapper = PathMapper::new(rust_rules) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + Ok(Self { + inner: HostContext::WithRules(Arc::new(mapper)), + }) } /// Whether this host context registers any host-context @@ -263,7 +270,7 @@ impl PyHostContext { HostContext::None => "HostContext.none()".to_string(), HostContext::Unresolved => "HostContext.unresolved()".to_string(), HostContext::WithRules(rules) => { - format!("HostContext.with_rules(<{} rule(s)>)", rules.len()) + format!("HostContext.with_rules(<{} rule(s)>)", rules.rules().len()) } } } @@ -306,6 +313,7 @@ impl PyHostContext { HostContext::WithRules(rules) => { let f = cls.getattr("with_rules")?; let py_rules: Vec = rules + .rules() .iter() .cloned() .map(|inner| PyPathMappingRule { inner }) @@ -328,8 +336,9 @@ fn host_context_eq(a: &HostContext, b: &HostContext) -> bool { match (a, b) { (HostContext::None, HostContext::None) => true, (HostContext::Unresolved, HostContext::Unresolved) => true, - (HostContext::WithRules(la), HostContext::WithRules(lb)) => { + (HostContext::WithRules(ma), HostContext::WithRules(mb)) => { // Compare rule-by-rule; `Arc` identity is irrelevant. + let (la, lb) = (ma.rules(), mb.rules()); la.len() == lb.len() && la.iter().zip(lb.iter()).all(|(ra, rb)| { ra.source_path_format == rb.source_path_format @@ -347,7 +356,8 @@ fn host_context_hash(hc: &HostContext) -> u64 { match hc { HostContext::None => 0u8.hash(&mut h), HostContext::Unresolved => 1u8.hash(&mut h), - HostContext::WithRules(rules) => { + HostContext::WithRules(mapper) => { + let rules = mapper.rules(); 2u8.hash(&mut h); (rules.len() as u64).hash(&mut h); for r in rules.iter() { diff --git a/rust-bindings/src/expr/range_expr.rs b/rust-bindings/src/expr/range_expr.rs index da2eca3f..b3900f26 100644 --- a/rust-bindings/src/expr/range_expr.rs +++ b/rust-bindings/src/expr/range_expr.rs @@ -54,8 +54,8 @@ impl PyIntRange { self.inner.step() } - fn __len__(&self) -> usize { - self.inner.len() + fn __len__(&self) -> PyResult { + crate::len_to_usize(self.inner.len()) } fn __contains__(&self, value: i64) -> bool { @@ -205,8 +205,8 @@ impl PyRangeExpr { .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Range expression is empty")) } - fn __len__(&self) -> usize { - self.inner.len() + fn __len__(&self) -> PyResult { + crate::len_to_usize(self.inner.len()) } fn __eq__(&self, other: &PyRangeExpr) -> bool { @@ -218,13 +218,13 @@ impl PyRangeExpr { } fn __getitem__(&self, index: isize) -> PyResult { - let len = self.inner.len(); + let len = i128::from(self.inner.len()); let idx = if index < 0 { - len as isize + index + len + index as i128 } else { - index + index as i128 }; - if idx < 0 || idx as usize >= len { + if idx < 0 || idx >= len { return Err(pyo3::exceptions::PyIndexError::new_err( "index out of range", )); diff --git a/rust-bindings/src/lib.rs b/rust-bindings/src/lib.rs index b90f334f..a7f87ca5 100644 --- a/rust-bindings/src/lib.rs +++ b/rust-bindings/src/lib.rs @@ -48,6 +48,18 @@ fn register_renamed_exception( Ok(()) } +/// Convert a `u64` count from the openjd crates to the `usize` that +/// `__len__` must return. The crates keep counts 64-bit on every target, +/// so on a 32-bit target a count above `usize::MAX` raises `OverflowError`, +/// which is what Python's `len()` raises for a length it cannot represent. +pub(crate) fn len_to_usize(len: u64) -> PyResult { + usize::try_from(len).map_err(|_| { + pyo3::exceptions::PyOverflowError::new_err( + "length does not fit in this platform's Py_ssize_t", + ) + }) +} + // ── Module ── #[pymodule] diff --git a/rust-bindings/src/model/step_param_space.rs b/rust-bindings/src/model/step_param_space.rs index 5f0df8e7..f5618441 100644 --- a/rust-bindings/src/model/step_param_space.rs +++ b/rust-bindings/src/model/step_param_space.rs @@ -109,13 +109,13 @@ pub(crate) struct PyStepParameterSpaceIterator { space: StepParameterSpace, /// Cached length captured at construction time. Used by `__len__` /// for non-adaptive spaces (adaptive spaces raise instead). - len: usize, + len: u64, /// Cached parameter names captured at construction time. names: HashSet, /// The persistent iterator that backs `__next__`, `reset_iter`, /// `__contains__`, and `chunks_default_task_count` (getter and /// setter). Holding it across calls is what lets the setter - /// actually mutate state — the underlying `Arc` for + /// actually mutate state — the underlying `Arc` for /// adaptive chunking lives inside this iterator. /// /// `Mutex` is required because pyclass types must be `Sync`. The @@ -127,7 +127,7 @@ pub(crate) struct PyStepParameterSpaceIterator { /// so `__getitem__` can rebuild an equivalent iterator. Without it, /// random access would silently fall back to the template's own /// `defaultTaskCount` and disagree with iteration. - chunk_override: Option, + chunk_override: Option, } #[cfg_attr(feature = "stub-gen", gen_stub_pymethods)] @@ -153,20 +153,20 @@ impl PyStepParameterSpaceIterator { space: Option<&PyStepParameterSpace>, chunks_task_count_override: Option, ) -> PyResult { - // Taken as i64, not usize, so a negative value reaches this check instead of + // Taken as i64, not u64, so a negative value reaches this check instead of // failing pyo3's unsigned extraction with OverflowError. Every non-positive // value should report the same ValueError the message and the spec promise. // // Rejected rather than clamped: openjd-model applies `.max(1)` to the override, // so 0 would silently mean 1. The `chunks_default_task_count` setter already // refuses 0, and the two should not disagree. - let chunks_task_count_override: Option = match chunks_task_count_override { + let chunks_task_count_override: Option = match chunks_task_count_override { Some(n) if n <= 0 => { return Err(pyo3::exceptions::PyValueError::new_err( "chunks_task_count_override must be a positive integer.", )); } - Some(n) => Some(n as usize), + Some(n) => Some(n as u64), None => None, }; let ps = if let Some(s) = space { @@ -210,7 +210,7 @@ impl PyStepParameterSpaceIterator { "Length is not available because the parameter space uses adaptive chunking.", )); } - Ok(self.len) + crate::len_to_usize(self.len) } fn __getitem__(&self, py: Python<'_>, index: isize) -> PyResult> { @@ -235,15 +235,15 @@ impl PyStepParameterSpaceIterator { )); } let idx = if index < 0 { - let adjusted = self.len as isize + index; + let adjusted = i128::from(self.len) + index as i128; if adjusted < 0 { return Err(pyo3::exceptions::PyIndexError::new_err( "index out of range", )); } - adjusted as usize + adjusted as u64 } else { - index as usize + index as u64 }; match iter.get(idx) { Some(params) => task_param_set_to_py(py, ¶ms), @@ -312,13 +312,13 @@ impl PyStepParameterSpaceIterator { } #[getter] - fn chunks_default_task_count(&self) -> Option { + fn chunks_default_task_count(&self) -> Option { let iter = lock_recover(&self.iter); iter.chunks_default_task_count() } #[setter] - fn set_chunks_default_task_count(&self, value: usize) -> PyResult<()> { + fn set_chunks_default_task_count(&self, value: u64) -> PyResult<()> { if value == 0 { return Err(pyo3::exceptions::PyValueError::new_err( "chunks_default_task_count must be a positive integer.", diff --git a/rust-bindings/src/model/step_param_space_def.rs b/rust-bindings/src/model/step_param_space_def.rs index 48aa9f7f..bace5b16 100644 --- a/rust-bindings/src/model/step_param_space_def.rs +++ b/rust-bindings/src/model/step_param_space_def.rs @@ -10,7 +10,7 @@ //! //! Five typed pyclasses mirror the `TaskParameterDefinition` variants: //! -//! * `IntTaskParameterDefinition` — `range: list[int] | FormatString` +//! * `IntTaskParameterDefinition` — `range: list[int | FormatString] | FormatString` //! * `FloatTaskParameterDefinition` — `range: list[float | FormatString] | FormatString` //! * `StringTaskParameterDefinition` — `range: list[FormatString] | FormatString` //! * `PathTaskParameterDefinition` — `range: list[FormatString] | FormatString` @@ -76,14 +76,14 @@ fn float_range_item_to_py<'py>(py: Python<'py>, v: &FloatRangeItem) -> PyResult< } } -/// Convert an `IntRange` to a Python value (`list[int]` or -/// `FormatString`). +/// Convert an `IntRange` to a Python value (`list[int | FormatString]` +/// or `FormatString`). fn int_range_to_py<'py>(py: Python<'py>, r: &IntRange) -> PyResult> { match r { IntRange::List(items) => { let py_list = PyList::empty(py); for item in items { - py_list.append(item.0)?; + py_list.append(int_or_fs_to_py(py, item)?)?; } Ok(py_list.into_any()) } diff --git a/rust-bindings/src/model/task_parameter.rs b/rust-bindings/src/model/task_parameter.rs index 57cc1868..54327c36 100644 --- a/rust-bindings/src/model/task_parameter.rs +++ b/rust-bindings/src/model/task_parameter.rs @@ -73,8 +73,8 @@ impl PyTaskChunksDefinition { #[new] #[pyo3(signature = (*, default_task_count, target_runtime_seconds=None, range_constraint))] fn new( - default_task_count: usize, - target_runtime_seconds: Option, + default_task_count: u64, + target_runtime_seconds: Option, range_constraint: &str, ) -> PyResult { let constraint = match range_constraint { @@ -96,12 +96,12 @@ impl PyTaskChunksDefinition { } #[getter] - fn default_task_count(&self) -> usize { + fn default_task_count(&self) -> u64 { self.inner.default_task_count } #[getter] - fn target_runtime_seconds(&self) -> Option { + fn target_runtime_seconds(&self) -> Option { self.inner.target_runtime_seconds } diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index 61e3c492..37e36e5b 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -422,8 +422,9 @@ impl PySession { /// Extend the session's path mapping rules with additional rules. /// - /// Rules are stored sorted by source-path length (longest first) so that - /// the most specific rule matches first during FormatString resolution. + /// When several rules match a path, the one whose source path has the + /// most components wins, and between rules for the same location the + /// one given last wins, wherever it sits in the list. /// /// Consumers (notably the Deadline Cloud worker agent's Job Attachments /// download action) call this after an action assigns additional path @@ -431,7 +432,9 @@ impl PySession { /// /// Returns an error if an action is currently in-flight (the underlying /// Session has been taken by a background thread). Call this between - /// actions. + /// actions. Raises `SessionError`, leaving the rules unchanged, when an + /// added rule is invalid, for example a POSIX or WINDOWS `source_path` + /// that is not absolute. fn extend_path_mapping_rules( &self, additional: Vec, @@ -444,8 +447,9 @@ impl PySession { })?; let rules: Vec = additional.into_iter().map(|p| p.inner).collect(); - session.extend_path_mapping_rules(rules); - Ok(()) + session + .extend_path_mapping_rules(rules) + .map_err(session_err_to_py) } // ── Non-blocking action methods ── diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index 63f0a72e..55a3bdc0 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -363,7 +363,18 @@ HostContext.unresolved().is_enabled() # True HostContext.unresolved().is_unresolved() # True — uses stub implementations HostContext.with_rules([]).is_enabled() # True — empty rules ≠ no host context HostContext.with_rules([]).is_unresolved() # False — uses real implementations +``` + +`HostContext.with_rules` raises `ValueError` when a rule is invalid, +such as a POSIX or WINDOWS rule whose `source_path` is not absolute; +the message names the rule's index, for example `path mapping rule 1: +source_path 'rel' must be an absolute POSIX path, starting with '/'`. +When several rules match a path, the one whose source path has the +most names applies, wherever it is in the list, and between rules for +the same location the last one given applies (How Jobs Are Run, "Path +Mapping"). The order of `rules` matters only for that tie. +```python # Inspecting a profile profile.revision # ExprRevision.V2026_02 profile.extensions # [] today @@ -635,6 +646,10 @@ matched, result = rule.apply(path="/mnt/shared/project/file.exr") matched, result = rule.apply(path="/other/path") # matched=False, result="/other/path" +# A rule whose POSIX or WINDOWS source_path is not absolute can be +# constructed, but apply() raises ValueError for it: +# "source_path 'rel' must be an absolute POSIX path, starting with '/'" + # Serialization d = rule.to_dict() # {"source_path_format": "POSIX", "source_path": "/mnt/shared", "destination_path": "/local/cache"} diff --git a/specs/python-model-interface.md b/specs/python-model-interface.md index e297e53f..701c3d44 100644 --- a/specs/python-model-interface.md +++ b/specs/python-model-interface.md @@ -1028,11 +1028,11 @@ underlying `template::TaskParameterDefinition` enum 1:1: | Variant | Pyclass | `range` element type | |---|---|---| -| `INT` | `IntTaskParameterDefinition` | `list[int]` or `FormatString` | +| `INT` | `IntTaskParameterDefinition` | `list[int \| FormatString]` or `FormatString` | | `FLOAT` | `FloatTaskParameterDefinition` | `list[float \| FormatString]` or `FormatString` | | `STRING` | `StringTaskParameterDefinition` | `list[FormatString]` or `FormatString` | | `PATH` | `PathTaskParameterDefinition` | `list[FormatString]` or `FormatString` | -| `CHUNK[INT]` | `ChunkIntTaskParameterDefinition` | `list[int]` or `FormatString` | +| `CHUNK[INT]` | `ChunkIntTaskParameterDefinition` | `list[int \| FormatString]` or `FormatString` | Common attributes on every variant: @@ -1048,10 +1048,13 @@ For lists where the element type is `int`/`float`/`FormatString`, the list-form vs format-string-form is dispatched by the binding: the `.range` getter returns either a Python list (literal range) or a `FormatString` (e.g. `"1-10:2"`, possibly carrying a -`{{Param.X}}` interpolation under the EXPR extension). Only `INT` and -`CHUNK[INT]` accept the list-form `[1, 2, 3]`; the others always carry -`FormatString` elements (which may themselves be literal or -interpolating). +`{{Param.X}}` interpolation under the EXPR extension). An `INT` or +`CHUNK[INT]` list element is a `FormatString` when it holds an +interpolation, such as `"{{Param.Frame}}"`, and an `int` otherwise +(`"5"` reads as `5`). A `FLOAT` list element is a `float` when the +template wrote a number and a `FormatString` when it wrote a string. +`STRING` and `PATH` elements are always `FormatString` (which may +itself be literal or interpolating). The `CHUNK[INT]` variant additionally exposes: @@ -1233,7 +1236,10 @@ Iteration observes it, and `len()` counts the overridden granularity. The rendered form of a chunk depends on `rangeConstraint`, which matters because these are strings a consumer parses and may feed back through `__contains__`. A single-task chunk is `"1-1"` under `CONTIGUOUS` and a -bare `"1"` under `NONCONTIGUOUS`. +bare `"1"` under `NONCONTIGUOUS`. A `NONCONTIGUOUS` chunk is written in +the canonical form of Template Schemas §3.4.1.1.1, so the values +`[1, 2, 4, 6]` render `"1,2-6:2"`. The pure-Python reference renders +`"1,2,4,6"` there. It is currently the only way to change the chunk size of a *static* chunked space: the `chunks_default_task_count` setter accepts adaptive diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index 243c6182..d4d04f61 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -175,6 +175,14 @@ boundary for those caps: a worker can run a job that never passed through the validating process, so a session that is given no limits enforces nothing beyond the spec. +`path_mapping_rules`, and the rules given to +`extend_path_mapping_rules`, must each have an absolute POSIX or +WINDOWS `source_path` (How Jobs Are Run, "Path Mapping"). A set with +an invalid rule raises `SessionError`, and `extend_path_mapping_rules` +leaves the existing rules unchanged. When several rules match a path, +the one whose source path has the most names applies, wherever it is +in the list. + ### SessionUser (stays in Python, passed to Rust) The `SessionUser` types involve OS-specific credential validation (sudo diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index 5bb5740a..d80733ba 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -1057,6 +1057,10 @@ class HostContext: host context (`apply_path_mapping` is registered, returns the path unchanged). Passing zero rules is *not* the same as passing no host context at all. + + Raises `ValueError` when a rule is invalid, for example a POSIX or + WINDOWS `source_path` that is not absolute. The message names the + rule's index in `rules`. """ def is_enabled(self) -> builtins.bool: @@ -1913,7 +1917,14 @@ class PathMappingRule: def apply( self, *, path: builtins.str, output_format: typing.Optional[PathFormat] = None - ) -> tuple[builtins.bool, builtins.str]: ... + ) -> tuple[builtins.bool, builtins.str]: + r""" + Apply this rule to `path`. Returns `(True, mapped)` when the rule + matches and `(False, path)` when it does not. Raises `ValueError` + when the rule itself is invalid, for example a POSIX or WINDOWS + `source_path` that is not absolute. + """ + def to_dict(self) -> builtins.dict[builtins.str, builtins.str]: ... @staticmethod def from_dict(d: dict) -> PathMappingRule: ... @@ -2178,8 +2189,9 @@ class Session: r""" Extend the session's path mapping rules with additional rules. - Rules are stored sorted by source-path length (longest first) so that - the most specific rule matches first during FormatString resolution. + When several rules match a path, the one whose source path has the + most components wins, and between rules for the same location the + one given last wins, wherever it sits in the list. Consumers (notably the Deadline Cloud worker agent's Job Attachments download action) call this after an action assigns additional path @@ -2187,7 +2199,9 @@ class Session: Returns an error if an action is currently in-flight (the underlying Session has been taken by a background thread). Call this between - actions. + actions. Raises `SessionError`, leaving the rules unchanged, when an + added rule is invalid, for example a POSIX or WINDOWS `source_path` + that is not absolute. """ def enter_environment( diff --git a/test/openjd/expr/test_path_mapping.py b/test/openjd/expr/test_path_mapping.py index be8b97e0..d416cdd5 100644 --- a/test/openjd/expr/test_path_mapping.py +++ b/test/openjd/expr/test_path_mapping.py @@ -215,6 +215,59 @@ def test_custom_scheme(self, tmp_path) -> None: class TestPathMappingRuleValidation: """Tests for from_dict validation shared across formats.""" + @pytest.mark.parametrize( + "source_path_format, source_path, expected", + [ + pytest.param( + PathFormat.POSIX, + "rel/a", + "source_path 'rel/a' must be an absolute POSIX path, starting with '/'", + id="relative POSIX", + ), + pytest.param( + PathFormat.WINDOWS, + "a\\b", + "source_path 'a\\b' must be an absolute WINDOWS path, starting with a drive " + "such as 'C:\\' or a UNC share such as '\\\\server\\share'", + id="relative WINDOWS", + ), + ], + ) + def test_apply_rejects_a_relative_source_path( + self, source_path_format: PathFormat, source_path: str, expected: str + ) -> None: + """Since openjd-expr 0.14.0 (openjd-rs#472) a rule must have an + absolute source path (How Jobs Are Run, "Applying Path Mapping Rules + within a Job Template", item 1), and applying one that doesn't raises. + Before, such a rule matched relative paths. The pure-Python reference + in openjd-sessions still accepts it.""" + rule = PathMappingRule( + source_path_format=source_path_format, + source_path=source_path, + destination_path="/b", + ) + with pytest.raises(ValueError) as exc_info: + rule.apply(path="rel/a/x") + assert str(exc_info.value) == expected + + def test_with_rules_rejects_a_relative_source_path(self) -> None: + """``HostContext.with_rules`` checks every rule, and the message + names the rule's index.""" + rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/a", destination_path="/b" + ), + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="rel", destination_path="/b" + ), + ] + with pytest.raises(ValueError) as exc_info: + HostContext.with_rules(rules) + assert str(exc_info.value) == ( + "path mapping rule 1: source_path 'rel' must be an absolute POSIX path, " + "starting with '/'" + ) + class TestPathMappingRuleSerialization: """Tests for from_dict/to_dict serialization.""" @@ -572,6 +625,34 @@ def test_format_string_resolve_string_applies_rules(self, tmp_path) -> None: result = fs.resolve_string(SymbolTable({}), profile=self._profile_with_rule(dest)) assert result == dest + os.sep + "file.exr" + @pytest.mark.parametrize("reverse", [False, True], ids=["shorter first", "longer first"]) + def test_the_rule_with_the_most_names_wins_in_any_order(self, tmp_path, reverse) -> None: + """Since openjd-expr 0.14.0 (openjd-rs#472) the rule whose source + path has the most names applies, wherever it is in the list (How + Jobs Are Run, "Applying Path Mapping Rules within a Job Template", + item 3). Before, ``with_rules`` used the first rule that matched.""" + from openjd.expr import parse_expression + + short_dest = str(tmp_path / "short") + long_dest = str(tmp_path / "long") + rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path="/mnt", + destination_path=short_dest, + ), + PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path="/mnt/shared", + destination_path=long_dest, + ), + ] + if reverse: + rules.reverse() + profile = ExprProfile().with_host_context(HostContext.with_rules(rules)) + parsed = parse_expression("apply_path_mapping('/mnt/shared/file.exr')") + assert parsed.evaluate(profile=profile).item() == long_dest + os.sep + "file.exr" + def test_format_string_resolve_applies_rules(self, tmp_path) -> None: from openjd.expr import FormatString, SymbolTable diff --git a/test/openjd/expr/test_paths.py b/test/openjd/expr/test_paths.py index 2ae13366..0787d5e7 100644 --- a/test/openjd/expr/test_paths.py +++ b/test/openjd/expr/test_paths.py @@ -4,7 +4,7 @@ import sys import pytest -from pathlib import PurePath +from pathlib import PurePath, PurePosixPath from openjd.expr import evaluate_expression, ExprValue, ExpressionError, SymbolTable from openjd.expr import PathFormat @@ -71,6 +71,24 @@ def test_path_join(self, tmp_path) -> None: result = evaluate_expression("Dir / 'subdir' / 'file.txt'", values=symtab) assert str(result) == str(d / "subdir" / "file.txt") + @pytest.mark.parametrize( + "left, right, expected", + [ + pytest.param("", "a", "a", id="empty left"), + pytest.param(".", "a", "a", id="dot left"), + pytest.param("a", "", "a", id="empty right"), + pytest.param("/x", "", "/x", id="empty right after a root"), + pytest.param("", "", ".", id="both empty"), + ], + ) + def test_path_join_with_an_empty_operand(self, left: str, right: str, expected: str) -> None: + """An empty operand gives the other operand, and two give ``.``, as + ``pathlib`` does (Expression Language §2.1.5). openjd-expr 0.14.0 + (openjd-rs#516); before, ``path('') / 'a'`` gave ``./a`` and + ``path('a') / ''`` gave ``a/``.""" + result = evaluate_expression(f"path('{left}') / '{right}'", path_format=PathFormat.POSIX) + assert str(result) == expected == str(PurePosixPath(left) / right) + def test_path_concat(self, tmp_path) -> None: p = tmp_path / "output" / "file" symtab = SymbolTable({"P": _p(p)}) diff --git a/test/openjd/expr/test_unresolved_eval.py b/test/openjd/expr/test_unresolved_eval.py index 1242ee55..03eca8ab 100644 --- a/test/openjd/expr/test_unresolved_eval.py +++ b/test/openjd/expr/test_unresolved_eval.py @@ -361,31 +361,37 @@ def test_empty_list_unchanged(self) -> None: class TestUnknownListComprehensions: - """Test list comprehension with unresolved iterables.""" + """Test list comprehension with unresolved iterables. + + Since openjd-expr 0.13.0 (openjd-rs#462) the element type is nullable, + ``list[T?]`` rather than ``list[T]``: a run may keep no element, and the + empty list has type ``list[nulltype]`` (Expression Language §1.3.7), so the + unresolved type has to cover it. + """ def test_unknown_list_iterable(self) -> None: - """[x for x in unresolved[list[int]]] -> unresolved[list[int]].""" + """[x for x in unresolved[list[int]]] -> unresolved[list[int?]].""" values = SymbolTable({"X": ExprValue.unresolved("list[int]")}) result = evaluate_expression("[x for x in X]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_unknown_list_with_body_expr(self) -> None: - """[x + 1 for x in unresolved[list[int]]] -> unresolved[list[int]].""" + """[x + 1 for x in unresolved[list[int]]] -> unresolved[list[int?]].""" values = SymbolTable({"X": ExprValue.unresolved("list[int]")}) result = evaluate_expression("[x + 1 for x in X]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_unknown_range_iterable(self) -> None: - """[x for x in unresolved[range_expr]] -> unresolved[list[int]].""" + """[x for x in unresolved[range_expr]] -> unresolved[list[int?]].""" values = SymbolTable({"X": ExprValue.unresolved("range_expr")}) result = evaluate_expression("[x for x in X]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_unknown_iterable_with_transform(self) -> None: - """[string(x) for x in unresolved[list[int]]] -> unresolved[list[string]].""" + """[string(x) for x in unresolved[list[int]]] -> unresolved[list[string?]].""" values = SymbolTable({"X": ExprValue.unresolved("list[int]")}) result = evaluate_expression("[string(x) for x in X]", values=values) - assert result.type == ExprType("unresolved[list[string]]") + assert result.type == ExprType("unresolved[list[string?]]") class TestUnknownCoercion: @@ -740,18 +746,19 @@ class TestConcreteIterableUnresolvedFilter: the one stage where the iterable is concrete and the filter is not. Extends ``TestUnknownListComprehensions`` above, which covers the - unresolved-iterable path. + unresolved-iterable path. The element type is nullable for the reason given + there. """ def test_concrete_list_iterable(self) -> None: values = SymbolTable({"Skip": ExprValue.unresolved(ExprType("string"))}) result = evaluate_expression("[f for f in 'a,b,c'.split(',') if f != Skip]", values=values) - assert result.type == ExprType("unresolved[list[string]]") + assert result.type == ExprType("unresolved[list[string?]]") def test_concrete_range_iterable(self) -> None: values = SymbolTable({"N": ExprValue.unresolved(ExprType("int"))}) result = evaluate_expression("[x * 2 for x in range(3) if x > N]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_the_body_type_is_derived_under_an_unresolved_loop_variable(self) -> None: """``10 // 0`` would raise if the body were evaluated on the concrete element @@ -761,7 +768,7 @@ def test_the_body_type_is_derived_under_an_unresolved_loop_variable(self) -> Non each element".""" values = SymbolTable({"N": ExprValue.unresolved(ExprType("int"))}) result = evaluate_expression("[10 // x for x in [0, 2] if x > N]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_a_filter_short_circuiting_before_the_unresolved_term(self) -> None: """For ``x = 0`` the filter is decided concretely (``0 > 0`` is false); for @@ -769,12 +776,14 @@ def test_a_filter_short_circuiting_before_the_unresolved_term(self) -> None: rather than returned as a partial list.""" values = SymbolTable({"B": ExprValue.unresolved(ExprType("bool"))}) result = evaluate_expression("[x for x in [0, 1, 2] if x > 0 and B]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") def test_a_filter_that_can_never_be_a_boolean_is_still_rejected(self) -> None: """Negative control. The change is not "stop checking the filter type" — a - filter whose type can never be a boolean errors on both paths. A mutation that - widened the fix to swallow this would pass every case above.""" + filter whose type can never be a boolean errors over a concrete, non-empty + iterable. A mutation that widened the fix to swallow this would pass every + case above. Over an unresolved iterable see + ``test_a_never_boolean_filter_over_an_unresolved_iterable_keeps_nothing``.""" values = SymbolTable({"Skip": ExprValue.unresolved(ExprType("string"))}) with pytest.raises(ExpressionError) as exc_info: evaluate_expression("[x for x in [1, 2] if Skip]", values=values) @@ -787,7 +796,19 @@ def test_the_unresolved_iterable_path_still_concludes_unresolved(self) -> None: {"X": ExprValue.unresolved("list[int]"), "N": ExprValue.unresolved("int")} ) result = evaluate_expression("[x for x in X if x > N]", values=values) - assert result.type == ExprType("unresolved[list[int]]") + assert result.type == ExprType("unresolved[list[int?]]") + + def test_a_never_boolean_filter_over_an_unresolved_iterable_keeps_nothing(self) -> None: + """Since openjd-expr 0.13.0 (openjd-rs#462), a filter that can never be a + boolean fails every run that evaluates it, so only a run that keeps no + element succeeds, and the result is the empty list's type rather than an + error (Expression Language §1.3.7). Before, this raised + ``List comprehension filter must be a boolean, got int``.""" + values = SymbolTable( + {"L": ExprValue.unresolved("list[int]"), "P": ExprValue.unresolved("int")} + ) + result = evaluate_expression("[x for x in L if P]", values=values) + assert result.type == ExprType("unresolved[list[nulltype]]") class TestBoolOpBudgetErrorsPropagate: diff --git a/test/openjd/model_v1/test_known_gaps.py b/test/openjd/model_v1/test_known_gaps.py index a6e18b17..438527b3 100644 --- a/test/openjd/model_v1/test_known_gaps.py +++ b/test/openjd/model_v1/test_known_gaps.py @@ -93,3 +93,30 @@ def test_v0_joins_list_path_defaults_to_the_template_directory() -> None: def test_v0_normalizes_a_relative_submitted_path() -> None: out = _v0_preprocess({"name": "P", "type": "PATH"}, {"P": "sub/../other"}) assert out["P"] == "/tmp/cwd/other" + + +# ── v0 writes a NONCONTIGUOUS chunk outside canonical form ── +# +# This runs v0, not the binding. Template Schemas §3.4.1.5 writes a NONCONTIGUOUS +# chunk in the canonical form of §3.4.1.1.1, which keeps a run only when it has +# three or more values, so ``[1, 2, 4, 6]`` is ``1,2-6:2``. v0's +# ``IntRangeExpr.from_list`` commits to the first gap as the step and writes +# ``1,2,4,6``. openjd-model 0.14.0 (openjd-rs#469) brought v1 to canonical form; +# ``TestNoncontiguousChunkRendering`` in ``test_step_param_space_iter.py`` pins +# the v1 side. + +_V0_CANONICAL_GAP = "v0 IntRangeExpr.from_list does not write canonical form (§3.4.1.1.1)" + + +@pytest.mark.xfail(strict=True, reason=_V0_CANONICAL_GAP) +@pytest.mark.parametrize( + "values, expected", + [ + pytest.param([1, 2, 4, 6], "1,2-6:2", id="pair then even progression"), + pytest.param([1, 3, 4, 5], "1,3-5", id="gap then consecutive run"), + ], +) +def test_v0_range_list_renders_in_canonical_form(values: list[int], expected: str) -> None: + from openjd.model._range_expr import IntRangeExpr + + assert str(IntRangeExpr.from_list(values)) == expected diff --git a/test/openjd/model_v1/test_parse.py b/test/openjd/model_v1/test_parse.py index 263bf9e4..e00cdef1 100644 --- a/test/openjd/model_v1/test_parse.py +++ b/test/openjd/model_v1/test_parse.py @@ -369,6 +369,25 @@ def test_invalid_yaml_raises(self) -> None: with pytest.raises(DecodeValidationError): decode_job_template_str("specificationVersion: nope") + @pytest.mark.parametrize("scalar", ["tRUE", "fALSE", "nULL"]) + def test_mixed_case_bool_and_null_spellings_are_strings(self, scalar: str) -> None: + """YAML 1.2 Core reads only ``true``/``True``/``TRUE`` (and the same for + ``false`` and ``null``) as booleans and null. Since openjd-model 0.15.0 + (openjd-rs#487, serde-saphyr 2.1) other spellings are strings, as they + are in the pure-Python reference. Before, ``tRUE`` was a boolean and + ``nULL`` was null.""" + t = decode_job_template_str( + "specificationVersion: jobtemplate-2023-09\n" + "name: T\n" + "parameterDefinitions:\n" + f"- {{name: P, type: STRING, default: {scalar}}}\n" + "steps:\n" + "- name: S\n" + " script: {actions: {onRun: {command: echo}}}\n" + ) + assert t.parameter_definitions is not None + assert t.parameter_definitions[0].default == scalar + class TestDecodeEnvironmentTemplateStr: """``decode_environment_template_str`` wrapper: parses YAML or diff --git a/test/openjd/model_v1/test_step_param_space_def.py b/test/openjd/model_v1/test_step_param_space_def.py index a26c0837..b609d736 100644 --- a/test/openjd/model_v1/test_step_param_space_def.py +++ b/test/openjd/model_v1/test_step_param_space_def.py @@ -181,6 +181,26 @@ def test_range_expression_string(self) -> None: assert isinstance(d.range, FormatString) assert d.range.raw() == "1-10:2" + def test_list_with_format_string(self) -> None: + """Since openjd-model 0.15.0 (openjd-rs#486) an INT list element may + be a format string, and it is exposed as a ``FormatString`` beside the + ``int`` literals, as the pure-Python reference exposes its + ``list[int | TaskParameterStringValue]``.""" + step = _decode_step( + parameter_definitions=[{"name": "X", "type": "INT"}], + parameter_space={ + "taskParameterDefinitions": [ + {"name": "F", "type": "INT", "range": [1, "{{Param.X}}", 3]}, + ], + }, + ) + d = step.parameter_space.task_parameter_definitions[0] + items = d.range + assert items[0] == 1 + assert isinstance(items[1], FormatString) + assert items[1].raw() == "{{Param.X}}" + assert items[2] == 3 + class TestFloatTaskParameterDefinition: def test_list_of_literals(self) -> None: @@ -198,9 +218,15 @@ def test_list_of_literals(self) -> None: def test_list_with_format_string(self) -> None: """``FloatRangeItem`` is a union — list elements can be - either literals (Python ``float``) or ``FormatString``s.""" + either literals (Python ``float``) or ``FormatString``s. + + ``Param.X`` is declared because since openjd-model 0.15.0 + (openjd-rs#486) a FLOAT list element's format string is checked at + decode time, as the pure-Python reference already did; see + ``test_list_element_with_undefined_reference_is_rejected``.""" step = _decode_step( extensions=["EXPR"], + parameter_definitions=[{"name": "X", "type": "FLOAT"}], parameter_space={ "taskParameterDefinitions": [ { @@ -218,6 +244,29 @@ def test_list_with_format_string(self) -> None: assert items[1].raw() == "{{Param.X}}" assert items[2] == 3.5 + def test_list_element_with_undefined_reference_is_rejected(self) -> None: + """openjd-model 0.15.0 (openjd-rs#486) checks the format strings in + FLOAT list elements at decode time. Before, an undefined reference + here decoded and failed only at job creation; the pure-Python + reference rejects it at decode time too.""" + with pytest.raises(ModelValidationError) as exc_info: + _decode_step( + extensions=["EXPR"], + parameter_space={ + "taskParameterDefinitions": [ + {"name": "F", "type": "FLOAT", "range": [1.5, "{{Param.X}}"]}, + ], + }, + ) + assert str(exc_info.value) == ( + "1 validation error for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[1]:\n" + "\tFailed to parse interpolation expression at [0, 11]. " + "Undefined variable: 'Param.X'.\n" + " Param.X\n" + " ~~~~~~^" + ) + class TestStringTaskParameterDefinition: def test_list_of_format_strings(self) -> None: @@ -285,6 +334,33 @@ def test_minimal_chunks(self) -> None: assert chunks.target_runtime_seconds is None assert chunks.range_constraint == "CONTIGUOUS" + def test_list_with_format_string(self) -> None: + """``CHUNK[INT]`` shares the INT range list type, so since + openjd-model 0.15.0 (openjd-rs#486) its list elements are also + ``int`` or ``FormatString``.""" + step = _decode_step( + extensions=["TASK_CHUNKING"], + parameter_definitions=[{"name": "X", "type": "INT"}], + parameter_space={ + "taskParameterDefinitions": [ + { + "name": "F", + "type": "CHUNK[INT]", + "range": ["{{Param.X}}", 7], + "chunks": { + "defaultTaskCount": 4, + "rangeConstraint": "NONCONTIGUOUS", + }, + } + ] + }, + ) + d = step.parameter_space.task_parameter_definitions[0] + items = d.range + assert isinstance(items[0], FormatString) + assert items[0].raw() == "{{Param.X}}" + assert items[1] == 7 + def test_chunks_with_target_runtime(self) -> None: """``targetRuntimeSeconds`` and a ``NONCONTIGUOUS`` constraint round-trip through the typed pyclass surface.""" diff --git a/test/openjd/model_v1/test_step_param_space_iter.py b/test/openjd/model_v1/test_step_param_space_iter.py index 957409fe..2a7d2eca 100644 --- a/test/openjd/model_v1/test_step_param_space_iter.py +++ b/test/openjd/model_v1/test_step_param_space_iter.py @@ -1011,16 +1011,19 @@ def test_yielded_values_round_trip_through_contains(self) -> None: class TestNoncontiguousChunkRendering: - """A NONCONTIGUOUS chunk renders its values the way the v0 reference's - ``IntRangeExpr.from_list`` does: commit to the first gap as the step, and - consume a consecutive pair atomically. openjd-model 0.8.0 (openjd-rs#398). - 0.7.1 used a compressor that only collapsed runs of three or more agreeing - gaps, so ``[1, 2, 4, 6]`` rendered as ``1,2-6:2``. Every spelling parses to - the same integers, which is why nothing caught the divergence; the rendered - text is what a task sees in ``{{Task.Param.Frame}}``. - - Expected values are the v0 reference's output for the same template, checked - by running both paths side by side when this test was written. + """A NONCONTIGUOUS chunk renders its values in the canonical form of + Template Schemas §3.4.1.1.1, which the chunk definition (§3.4.1.5) requires. + The rendered text is what a task sees in ``{{Task.Param.Frame}}``. + + openjd-model 0.14.0 (openjd-rs#469) writes range expressions in canonical + form, which starts a run at the first value not yet in one and keeps it only + when it has three or more values, so ``[1, 2, 4, 6]`` renders ``1,2-6:2``. + From openjd-model 0.8.0 (openjd-rs#398) until then, the binding matched the + pure-Python reference's ``IntRangeExpr.from_list``, which commits to the + first gap as the step and consumes a consecutive pair atomically, rendering + ``1,2,4,6``. The pure-Python reference still does that, so the first two + cases below differ from it; that is a gap in the reference, not here. + Every spelling parses to the same integers. """ @staticmethod @@ -1062,11 +1065,11 @@ def _frames(range_values: list[str], default_task_count: int) -> list[str]: @pytest.mark.parametrize( "range_values, expected", [ - pytest.param(["1", "2", "4", "6"], "1,2,4,6", id="pair then even progression"), - pytest.param(["1", "3", "4", "5"], "1,3,4,5", id="gap then consecutive run"), + pytest.param(["1", "2", "4", "6"], "1,2-6:2", id="pair then even progression"), + pytest.param(["1", "3", "4", "5"], "1,3-5", id="gap then consecutive run"), pytest.param(["7", "8"], "7,8", id="lone pair"), pytest.param(["1", "3", "5", "7"], "1-7:2", id="control: uniform step compresses"), ], ) - def test_chunk_renders_like_the_reference(self, range_values: list[str], expected: str) -> None: + def test_chunk_renders_in_canonical_form(self, range_values: list[str], expected: str) -> None: assert self._frames(range_values, default_task_count=4) == [expected] diff --git a/test/openjd/sessions/test_path_mapping_rules.py b/test/openjd/sessions/test_path_mapping_rules.py new file mode 100644 index 00000000..0fdb6ce9 --- /dev/null +++ b/test/openjd/sessions/test_path_mapping_rules.py @@ -0,0 +1,62 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``Session`` rejects an invalid path mapping rule. + +Since openjd-sessions 0.9.0 (openjd-rs#472) a session keeps its rules in a +``PathMapper``, which requires each POSIX or WINDOWS ``source_path`` to be +absolute (How Jobs Are Run, "Path Mapping"). Before, a relative rule was +accepted and matched relative paths. The rules are checked when the session is +constructed and when ``extend_path_mapping_rules`` adds to them. +""" + +from __future__ import annotations + +import time + +import pytest + +from openjd import _openjd_rs +from openjd._openjd_rs import Session +from openjd.expr import PathFormat, PathMappingRule + +# Registered at runtime by `register_renamed_exception`; the type stub doesn't +# declare the exception classes. +SessionError: type[Exception] = getattr(_openjd_rs, "SessionError") + +_GOOD = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/a", destination_path="/b" +) +_RELATIVE = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="rel", destination_path="/b" +) +_MESSAGE = ( + "Invalid path mapping rules: path mapping rule 1: source_path 'rel' must be an " + "absolute POSIX path, starting with '/'" +) + + +def test_construction_rejects_a_relative_source_path() -> None: + with pytest.raises(SessionError) as exc_info: + Session( + session_id=f"path-mapping-{time.time_ns()}", + job_parameter_values={}, + path_mapping_rules=[_GOOD, _RELATIVE], + ) + assert str(exc_info.value) == _MESSAGE + + +def test_extend_rejects_a_relative_source_path() -> None: + """The index counts from the first existing rule, so the one added rule + after one existing rule is rule 1.""" + session = Session( + session_id=f"path-mapping-{time.time_ns()}", + job_parameter_values={}, + path_mapping_rules=[_GOOD], + ) + try: + with pytest.raises(SessionError) as exc_info: + session.extend_path_mapping_rules([_RELATIVE]) + assert str(exc_info.value) == _MESSAGE + finally: + session.cleanup() From b906931d862c5df3e609144b7c7ba52650596f83 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:46:22 +0000 Subject: [PATCH 02/21] chore: Regenerate THIRD-PARTY-LICENSES.txt for the openjd-rs crate bump Regenerated with scripts/check_third_party_licenses.sh --update. Only the version lines of openjd-expr, openjd-model, openjd-sessions, serde-saphyr and granit-parser changed. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- THIRD-PARTY-LICENSES.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 273fb340..f1a83782 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -2739,13 +2739,13 @@ limitations under the License. ** arraydeque; version 0.5.1 -- https://crates.io/crates/arraydeque ** get-size-derive2; version 0.11.1 -- https://crates.io/crates/get-size-derive2 ** get-size2; version 0.11.1 -- https://crates.io/crates/get-size2 -** granit-parser; version 1.2.0 -- https://crates.io/crates/granit-parser +** granit-parser; version 2.1.0 -- https://crates.io/crates/granit-parser ** itoa; version 1.0.18 -- https://crates.io/crates/itoa ** libc; version 0.2.189 -- https://crates.io/crates/libc ** manyhow-macros; version 0.14.1 -- https://crates.io/crates/manyhow-macros -** openjd-expr; version 0.12.0 -- https://crates.io/crates/openjd-expr -** openjd-model; version 0.13.0 -- https://crates.io/crates/openjd-model -** openjd-sessions; version 0.8.1 -- https://crates.io/crates/openjd-sessions +** openjd-expr; version 0.14.0 -- https://crates.io/crates/openjd-expr +** openjd-model; version 0.15.0 -- https://crates.io/crates/openjd-model +** openjd-sessions; version 0.9.0 -- https://crates.io/crates/openjd-sessions ** pin-project-lite; version 0.2.17 -- https://crates.io/crates/pin-project-lite ** portable-atomic; version 1.15.0 -- https://crates.io/crates/portable-atomic ** proc-macro2; version 1.0.107 -- https://crates.io/crates/proc-macro2 @@ -2758,7 +2758,7 @@ limitations under the License. ** rustc-hash; version 2.1.3 -- https://crates.io/crates/rustc-hash ** rustversion; version 1.0.23 -- https://crates.io/crates/rustversion ** ryu; version 1.0.23 -- https://crates.io/crates/ryu -** serde-saphyr; version 1.2.0 -- https://crates.io/crates/serde-saphyr +** serde-saphyr; version 2.1.1 -- https://crates.io/crates/serde-saphyr ** serde; version 1.0.229 -- https://crates.io/crates/serde ** serde_core; version 1.0.229 -- https://crates.io/crates/serde_core ** serde_derive; version 1.0.229 -- https://crates.io/crates/serde_derive From 94ae92a9023a07889f8e7d4881011b791de5f8be Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:03:02 +0000 Subject: [PATCH 03/21] fix!: Cover more of the openjd-rs crate bump's behavior changes Review of the bump found visible changes that had no test or note: - openjd.expr.RangeExpr is kept in canonical form since openjd-expr 0.13.0 (openjd-rs#469): str, repr and ranges() give the canonical runs, and equality is by value. Every constructor caps the canonical form at 50,000 runs (openjd-rs#470). New tests pin these, and the strict xfail for the pure-Python IntRangeExpr also covers from_str. - Rules for the same location resolve last-wins (openjd-rs#472). A new test pins it. - An INT range list format string that doesn't resolve to an integer fails create_job (openjd-rs#486). A new test pins the message. - The pure-Python model's EXPR evaluation builds its profile with HostContext.with_rules, which now raises ValueError for a relative source path. Build the profile inside the existing try, so the failure reaches callers as the model's ExpressionError like any other evaluation error, and pin it. BREAKING CHANGE: str(), repr(), ranges() and == of openjd.expr.RangeExpr follow the canonical form, and a RangeExpr with more than 50,000 canonical runs raises RangeExprError. Between path mapping rules for the same location, the last one given applies. In the pure-Python model, an EXPR expression evaluated under a path mapping rule whose source path is not absolute raises ExpressionError. YAML scalars spelled tRUE, fALSE or nULL are strings. An unresolved list comprehension whose filter can never be a boolean gives list[nulltype] rather than raising. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- specs/python-expr-interface.md | 10 ++++ src/openjd/model/_format_strings/_nodes.py | 6 ++- test/openjd/expr/test_path_mapping.py | 22 ++++++++ test/openjd/expr/test_range_expr.py | 52 ++++++++++++++++++- .../model_v0/format_strings/test_expr_node.py | 27 ++++++++++ test/openjd/model_v1/test_create_job.py | 44 ++++++++++++++++ test/openjd/model_v1/test_known_gaps.py | 16 ++++++ 7 files changed, 175 insertions(+), 2 deletions(-) diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index 55a3bdc0..1a98a392 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -742,6 +742,16 @@ RangeExpr.from_list(["1", "2", "3"]) # 1-3 input string. Two `RangeExpr` values that compare equal also hash equal (suitable as `set` / `dict` keys). +A `RangeExpr` is kept in the canonical form of Template Schemas +§3.4.1.1.1, whatever elements the input used. `str`, `repr` and +`ranges()` give the canonical runs, so `RangeExpr("1,3,5,7,9")` is +`1-9:2` and `RangeExpr("1-2").ranges()` is two single-value ranges. +Two `RangeExpr`s are equal exactly when they have the same values. +Every constructor raises `RangeExprError` when the canonical form would +have more than 50,000 runs. The pure-Python +`openjd.model._range_expr.IntRangeExpr` follows the input's elements +instead. + ### `IntRange` A single contiguous integer range — the element type returned by diff --git a/src/openjd/model/_format_strings/_nodes.py b/src/openjd/model/_format_strings/_nodes.py index 8e74fdf6..750283b6 100644 --- a/src/openjd/model/_format_strings/_nodes.py +++ b/src/openjd/model/_format_strings/_nodes.py @@ -240,8 +240,12 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: values = symtab_to_expr_values( symtab, types=symtab.expr_types or None, path_format=path_format ) - profile = profile_for_symtab(symtab) try: + # Inside the try: ``HostContext.with_rules`` raises ``ValueError`` + # for an invalid path mapping rule, such as one whose source path + # is not absolute, and that reaches the caller as an expression + # error like any other. + profile = profile_for_symtab(symtab) return self._parsed.evaluate(values=values, profile=profile, path_format=path_format) except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error raise map_eval_error(exc) diff --git a/test/openjd/expr/test_path_mapping.py b/test/openjd/expr/test_path_mapping.py index d416cdd5..5906ecef 100644 --- a/test/openjd/expr/test_path_mapping.py +++ b/test/openjd/expr/test_path_mapping.py @@ -653,6 +653,28 @@ def test_the_rule_with_the_most_names_wins_in_any_order(self, tmp_path, reverse) parsed = parse_expression("apply_path_mapping('/mnt/shared/file.exr')") assert parsed.evaluate(profile=profile).item() == long_dest + os.sep + "file.exr" + def test_the_last_rule_for_the_same_location_wins(self, tmp_path) -> None: + """Rules whose source paths have the same components name the same + location, and the last one given applies (How Jobs Are Run, "Applying + Path Mapping Rules within a Job Template", item 3). openjd-expr 0.14.0 + (openjd-rs#472); before, the first one applied. ``/mnt`` and ``/mnt/`` + name the same location.""" + from openjd.expr import parse_expression + + first = str(tmp_path / "first") + last = str(tmp_path / "last") + rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt", destination_path=first + ), + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt/", destination_path=last + ), + ] + profile = ExprProfile().with_host_context(HostContext.with_rules(rules)) + parsed = parse_expression("apply_path_mapping('/mnt/file.exr')") + assert parsed.evaluate(profile=profile).item() == last + os.sep + "file.exr" + def test_format_string_resolve_applies_rules(self, tmp_path) -> None: from openjd.expr import FormatString, SymbolTable diff --git a/test/openjd/expr/test_range_expr.py b/test/openjd/expr/test_range_expr.py index 93e25821..aaeb1d79 100644 --- a/test/openjd/expr/test_range_expr.py +++ b/test/openjd/expr/test_range_expr.py @@ -4,7 +4,7 @@ import pytest from openjd.expr import evaluate_expression, SymbolTable, ExpressionError, TypeCode -from openjd.expr import RangeExpr, IntRange +from openjd.expr import RangeExpr, RangeExprError, IntRange class TestRangeExpr: @@ -184,6 +184,56 @@ def test_from_list_invalid_type_raises(self) -> None: RangeExpr.from_list([1.5]) +class TestRangeExprCanonicalForm: + """Since openjd-expr 0.13.0 (openjd-rs#469) a ``RangeExpr`` is kept and + written in the canonical form of Template Schemas §3.4.1.1.1, and two + compare equal exactly when they have the same values. Before, the string + and ``ranges()`` followed the input's elements, so ``"1,3,5,7,9"`` stayed + five runs and did not equal ``"1-9:2"``. The pure-Python + ``openjd.model._range_expr.IntRangeExpr`` still follows the input; see + ``test_v0_range_list_renders_in_canonical_form`` in + ``test/openjd/model_v1/test_known_gaps.py``.""" + + @pytest.mark.parametrize( + "text, expected", + [ + pytest.param("1,3,5,7,9", "1-9:2", id="singles join a run"), + pytest.param("1,2,4,6", "1,2-6:2", id="a pair leaves the run"), + pytest.param("1-10:2,11", "1-11:2", id="a run extends across elements"), + pytest.param("1-10:4", "1-9:4", id="the end is the last value"), + pytest.param("13-18,19", "13-19", id="adjacent elements merge"), + ], + ) + def test_str_is_canonical(self, text: str, expected: str) -> None: + assert str(RangeExpr(text)) == expected + assert repr(RangeExpr(text)) == f'RangeExpr("{expected}")' + + def test_ranges_are_the_canonical_runs(self) -> None: + """A run of two values is written as two single values, so ``"1-2"`` + has two ranges.""" + assert RangeExpr("1-2").ranges() == [IntRange(1, 1, 1), IntRange(2, 2, 1)] + assert RangeExpr("1,3,5,7,9").ranges() == [IntRange(1, 9, 2)] + + def test_equality_is_by_value(self) -> None: + a = RangeExpr("1,3,5,7,9") + b = RangeExpr("1-9:2") + assert a == b + assert hash(a) == hash(b) + + def test_from_list_caps_the_canonical_runs(self) -> None: + """openjd-expr 0.14.0 (openjd-rs#470) caps every constructor at 50,000 + runs of the canonical form. 25,000 pairs are 50,000 runs, and one more + value is one too many.""" + values = [v for i in range(25_000) for v in (i * 10, i * 10 + 1)] + assert len(RangeExpr.from_list(values).ranges()) == 50_000 + with pytest.raises(RangeExprError) as exc_info: + RangeExpr.from_list(values + [10**7]) + assert str(exc_info.value) == ( + "Range expression has too many sub-ranges in canonical form (50001); " + "maximum is 50000" + ) + + class TestExprValueRangeExprProtocols: """Test len, indexing, and iteration on ExprValue with range_expr type.""" diff --git a/test/openjd/model_v0/format_strings/test_expr_node.py b/test/openjd/model_v0/format_strings/test_expr_node.py index e54fa7fe..6976d463 100644 --- a/test/openjd/model_v0/format_strings/test_expr_node.py +++ b/test/openjd/model_v0/format_strings/test_expr_node.py @@ -355,6 +355,33 @@ def test_evaluate_value_returns_typed_engine_value(self): assert str(value) == "1.50" +class TestHostRules: + """A session sets ``expr_host_rules`` on the symbol table, which gives the + engine a ``HostContext.with_rules`` profile.""" + + def test_an_invalid_rule_raises_an_expression_error(self): + """Since openjd-expr 0.14.0 (openjd-rs#472) ``HostContext.with_rules`` + rejects a rule whose source path is not absolute (How Jobs Are Run, + "Path Mapping"), so every EXPR expression evaluated under such a set of + rules fails, even one that maps no path. The failure is the model's + ``ExpressionError``, not a bare ``ValueError``.""" + from openjd.expr import PathMappingRule + + st = SymbolTable() + st.expr_host_rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="rel", destination_path="/b" + ) + ] + node = parse_format_string_expr("1 + 2", context=_ctx(["EXPR"])) + with pytest.raises(ExpressionError) as excinfo: + node.evaluate(symtab=st) + assert str(excinfo.value) == ( + "path mapping rule 0: source_path 'rel' must be an absolute POSIX path, " + "starting with '/'" + ) + + class TestFormatStringResolveValue: """`FormatString.resolve_value` (RFC 0005/0006 typed resolution): a whole-field `{{ ... }}` EXPR expression resolves to the engine's typed diff --git a/test/openjd/model_v1/test_create_job.py b/test/openjd/model_v1/test_create_job.py index dc3b32a2..5607c8bb 100644 --- a/test/openjd/model_v1/test_create_job.py +++ b/test/openjd/model_v1/test_create_job.py @@ -2973,3 +2973,47 @@ def test_two_missing_extensions_are_listed_sorted(self) -> None: "missing EXPR, FEATURE_BUNDLE_1. An application that does not support an extension " "should reject the template at decode via its supported-extensions list." ) + + +class TestIntRangeListFormatStringElements: + """Since openjd-model 0.15.0 (openjd-rs#486) an INT range list element may + be a format string. ``create_job`` resolves it and requires an integer + (Template Schemas §3.4.1, Expression Language §1.3.2).""" + + @staticmethod + def _create(value: str) -> Any: + decoded = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": [{"name": "X", "type": "STRING"}], + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "F", "type": "INT", "range": [1, "{{Param.X}}"]} + ] + }, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + } + ) + return create_job(job_template=decoded, job_parameter_values={"X": value}) + + def test_an_integer_value_resolves(self) -> None: + job = self._create(" 7 ") + space = job.steps[0].parameterSpace + assert space is not None + assert space.taskParameterDefinitions["F"].range == [1, 7] + + @pytest.mark.parametrize("value", ["abc", "1.5"]) + def test_a_value_that_is_not_an_integer_fails(self, value: str) -> None: + from openjd.expr import ExpressionError + + with pytest.raises(ExpressionError) as excinfo: + self._create(value) + assert ( + str(excinfo.value) == f"Task parameter 'F' range[1]: value '{value}' is not an integer" + ) diff --git a/test/openjd/model_v1/test_known_gaps.py b/test/openjd/model_v1/test_known_gaps.py index 438527b3..1aa6f02e 100644 --- a/test/openjd/model_v1/test_known_gaps.py +++ b/test/openjd/model_v1/test_known_gaps.py @@ -120,3 +120,19 @@ def test_v0_range_list_renders_in_canonical_form(values: list[int], expected: st from openjd.model._range_expr import IntRangeExpr assert str(IntRangeExpr.from_list(values)) == expected + + +@pytest.mark.xfail(strict=True, reason=_V0_CANONICAL_GAP) +@pytest.mark.parametrize( + "text, expected", + [ + pytest.param("1,3,5,7,9", "1-9:2", id="singles join a run"), + pytest.param("1,2,4,6", "1,2-6:2", id="a pair leaves the run"), + ], +) +def test_v0_range_string_renders_in_canonical_form(text: str, expected: str) -> None: + """``openjd.expr.RangeExpr`` has written canonical form since openjd-expr + 0.13.0; ``TestRangeExprCanonicalForm`` in ``test/openjd/expr`` pins it.""" + from openjd.model._range_expr import IntRangeExpr + + assert str(IntRangeExpr.from_str(text)) == expected From 66d2767a6dff495ef957d98d1fb8d4927d144ddc Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:10:04 +0000 Subject: [PATCH 04/21] fix!: Pin the path length limit, separator and escape changes from the bump A second review of the bump found more visible changes with no test or note: - No separator is added after a path mapping destination that already ends with one (openjd-rs#483), so /dst/ maps /mnt/a/b to /dst/a/b. - A path longer than 32,768 bytes matches no rule, and a rule with a longer source or destination path is invalid (openjd-rs#472). - escape_format_string escapes a lone "{" before "}}", so "{}}" now round-trips (openjd-rs#515). New tests pin each one, and the expr and sessions specs describe them, including that a rule added by extend_path_mapping_rules replaces an existing rule for its location. The YAML scalar test said the change came from openjd-model 0.15.0. It comes from serde-saphyr 2.1, which Cargo.lock resolves now that openjd-model requires serde-saphyr 2, and which openjd-rs adopted in #487. Reword it. Use "names", as the specs do, in the extend_path_mapping_rules docstring, and name IntRangeExpr rather than only from_list in the known-gap reason. BREAKING CHANGE: a path longer than 32,768 bytes matches no path mapping rule, and a rule whose source or destination path is longer raises ValueError (SessionError in a Session). Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- rust-bindings/src/sessions/session.rs | 2 +- specs/python-expr-interface.md | 9 +++- specs/python-sessions-interface.md | 13 +++--- src/openjd/_openjd_rs.pyi | 2 +- test/openjd/expr/test_parse_expression.py | 10 +++++ test/openjd/expr/test_path_mapping.py | 55 +++++++++++++++++++++++ test/openjd/model_v1/test_known_gaps.py | 5 ++- test/openjd/model_v1/test_parse.py | 9 ++-- 8 files changed, 91 insertions(+), 14 deletions(-) diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index 37e36e5b..c7d95f3d 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -423,7 +423,7 @@ impl PySession { /// Extend the session's path mapping rules with additional rules. /// /// When several rules match a path, the one whose source path has the - /// most components wins, and between rules for the same location the + /// most names wins, and between rules for the same location the /// one given last wins, wherever it sits in the list. /// /// Consumers (notably the Deadline Cloud worker agent's Job Attachments diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index 1a98a392..170ff626 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -65,6 +65,8 @@ metered.operation_count # operations performed (>= 0) ### `escape_format_string` Escape `{{` and `}}` in a string for use as a literal in a format string. +A lone `{` just before `}}` is escaped too, so the result always parses +and resolves back to the input. ```python from openjd.expr import escape_format_string @@ -373,6 +375,9 @@ When several rules match a path, the one whose source path has the most names applies, wherever it is in the list, and between rules for the same location the last one given applies (How Jobs Are Run, "Path Mapping"). The order of `rules` matters only for that tie. +A `source_path` or `destination_path` longer than 32,768 bytes makes a +rule invalid, a path longer than that matches no rule, and the rules +together are limited to about 30 MiB of lookup structure. ```python # Inspecting a profile @@ -646,9 +651,11 @@ matched, result = rule.apply(path="/mnt/shared/project/file.exr") matched, result = rule.apply(path="/other/path") # matched=False, result="/other/path" -# A rule whose POSIX or WINDOWS source_path is not absolute can be +# A rule whose POSIX or WINDOWS source_path is not absolute, or whose +# source_path or destination_path is longer than 32,768 bytes, can be # constructed, but apply() raises ValueError for it: # "source_path 'rel' must be an absolute POSIX path, starting with '/'" +# A path longer than 32,768 bytes matches no rule. # Serialization d = rule.to_dict() diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index d4d04f61..901c4aa7 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -177,11 +177,14 @@ enforces nothing beyond the spec. `path_mapping_rules`, and the rules given to `extend_path_mapping_rules`, must each have an absolute POSIX or -WINDOWS `source_path` (How Jobs Are Run, "Path Mapping"). A set with -an invalid rule raises `SessionError`, and `extend_path_mapping_rules` -leaves the existing rules unchanged. When several rules match a path, -the one whose source path has the most names applies, wherever it is -in the list. +WINDOWS `source_path`, and no `source_path` or `destination_path` +longer than 32,768 bytes (How Jobs Are Run, "Path Mapping"). A set +with an invalid rule raises `SessionError`, and +`extend_path_mapping_rules` leaves the existing rules unchanged. When +several rules match a path, the one whose source path has the most +names applies, wherever it is in the list. Between rules for the same +location the last one applies, so a rule added by +`extend_path_mapping_rules` replaces an existing rule for its location. ### SessionUser (stays in Python, passed to Rust) diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index d80733ba..f877d908 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -2190,7 +2190,7 @@ class Session: Extend the session's path mapping rules with additional rules. When several rules match a path, the one whose source path has the - most components wins, and between rules for the same location the + most names wins, and between rules for the same location the one given last wins, wherever it sits in the list. Consumers (notably the Deadline Cloud worker agent's Job Attachments diff --git a/test/openjd/expr/test_parse_expression.py b/test/openjd/expr/test_parse_expression.py index 208c3182..b30c3a91 100644 --- a/test/openjd/expr/test_parse_expression.py +++ b/test/openjd/expr/test_parse_expression.py @@ -263,3 +263,13 @@ def test_escaped_braces(self): fs = FormatString(escaped) st = SymbolTable() assert fs.resolve_string(st) == "{{literal}}" + + @pytest.mark.parametrize("raw", ["{}}", "a{}}b", "{{}}}"]) + def test_escaped_brace_before_a_closing_pair(self, raw: str) -> None: + """A lone ``{`` just before ``}}`` is escaped too, so the result + parses and resolves to ``raw``. openjd-expr 0.14.0 (openjd-rs#515); + before, ``"{}}"`` gave ``{{{ "}" + "}" }}``, which failed to parse.""" + from openjd.expr import escape_format_string + + escaped = escape_format_string(raw) + assert FormatString(escaped).resolve_string(SymbolTable()) == raw diff --git a/test/openjd/expr/test_path_mapping.py b/test/openjd/expr/test_path_mapping.py index 5906ecef..252f0c1a 100644 --- a/test/openjd/expr/test_path_mapping.py +++ b/test/openjd/expr/test_path_mapping.py @@ -415,6 +415,61 @@ def test_windows_trailing_forward_slash(self, tmp_path) -> None: assert result.endswith("/") or result.endswith(os.sep) or result.endswith("\\") +class TestDestinationEndingInASeparator: + """Since openjd-expr 0.14.0 (openjd-rs#483) no separator is added after a + destination that already ends with one. Before, ``/dst/`` gave + ``/dst//a/b``. The pure-Python reference gives ``/dst/a/b`` too.""" + + def test_apply(self) -> None: + rule = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt", destination_path="/dst/" + ) + assert rule.apply(path="/mnt/a/b", output_format=PathFormat.POSIX) == (True, "/dst/a/b") + + def test_apply_path_mapping(self) -> None: + from openjd.expr import parse_expression + + rule = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt", destination_path="/dst/" + ) + profile = ExprProfile().with_host_context(HostContext.with_rules([rule])) + parsed = parse_expression("apply_path_mapping('/mnt/a/b')") + result = parsed.evaluate(profile=profile, path_format=PathFormat.POSIX) + assert result.item() == "/dst/a/b" + + +class TestPathLengthLimit: + """Since openjd-expr 0.14.0 (openjd-rs#472) a path or rule path longer than + 32 KiB (``MAX_PATH_LEN``, 32,768 bytes) is refused. A longer path matches + no rule, and a rule with a longer source or destination path is invalid. + Before, both mapped. The pure-Python reference still maps them.""" + + _RULE = PathMappingRule( + source_path_format=PathFormat.POSIX, source_path="/mnt", destination_path="/dst" + ) + + def test_a_path_at_the_limit_maps(self) -> None: + path = "/mnt/" + "a" * (32 * 1024 - len("/mnt/")) + matched, _ = self._RULE.apply(path=path, output_format=PathFormat.POSIX) + assert matched is True + + def test_a_longer_path_matches_no_rule(self) -> None: + path = "/mnt/" + "a" * (32 * 1024 - len("/mnt/") + 1) + assert self._RULE.apply(path=path) == (False, path) + + def test_a_longer_destination_path_is_invalid(self) -> None: + rule = PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path="/mnt", + destination_path="/" + "d" * (32 * 1024), + ) + with pytest.raises(ValueError) as exc_info: + rule.apply(path="/mnt/x") + assert str(exc_info.value) == ( + "destination_path is 32769 bytes long, more than the limit of 32768 bytes" + ) + + class TestFormatMismatch: """Tests that constructor rejects wrong path types for the format.""" diff --git a/test/openjd/model_v1/test_known_gaps.py b/test/openjd/model_v1/test_known_gaps.py index 1aa6f02e..9966817d 100644 --- a/test/openjd/model_v1/test_known_gaps.py +++ b/test/openjd/model_v1/test_known_gaps.py @@ -101,11 +101,12 @@ def test_v0_normalizes_a_relative_submitted_path() -> None: # chunk in the canonical form of §3.4.1.1.1, which keeps a run only when it has # three or more values, so ``[1, 2, 4, 6]`` is ``1,2-6:2``. v0's # ``IntRangeExpr.from_list`` commits to the first gap as the step and writes -# ``1,2,4,6``. openjd-model 0.14.0 (openjd-rs#469) brought v1 to canonical form; +# ``1,2,4,6``, and ``IntRangeExpr.from_str`` keeps the input's elements. +# openjd-model 0.14.0 (openjd-rs#469) brought v1 to canonical form; # ``TestNoncontiguousChunkRendering`` in ``test_step_param_space_iter.py`` pins # the v1 side. -_V0_CANONICAL_GAP = "v0 IntRangeExpr.from_list does not write canonical form (§3.4.1.1.1)" +_V0_CANONICAL_GAP = "v0 IntRangeExpr does not write canonical form (§3.4.1.1.1)" @pytest.mark.xfail(strict=True, reason=_V0_CANONICAL_GAP) diff --git a/test/openjd/model_v1/test_parse.py b/test/openjd/model_v1/test_parse.py index e00cdef1..27955922 100644 --- a/test/openjd/model_v1/test_parse.py +++ b/test/openjd/model_v1/test_parse.py @@ -372,10 +372,11 @@ def test_invalid_yaml_raises(self) -> None: @pytest.mark.parametrize("scalar", ["tRUE", "fALSE", "nULL"]) def test_mixed_case_bool_and_null_spellings_are_strings(self, scalar: str) -> None: """YAML 1.2 Core reads only ``true``/``True``/``TRUE`` (and the same for - ``false`` and ``null``) as booleans and null. Since openjd-model 0.15.0 - (openjd-rs#487, serde-saphyr 2.1) other spellings are strings, as they - are in the pure-Python reference. Before, ``tRUE`` was a boolean and - ``nULL`` was null.""" + ``false`` and ``null``) as booleans and null. With serde-saphyr 2.1, + which Cargo.lock resolves since the move to openjd-model 0.15.0 + (openjd-rs adopted it in #487), other spellings are strings, as they + are in the pure-Python reference. Before, with serde-saphyr 1.2, + ``tRUE`` was a boolean and ``nULL`` was null.""" t = decode_job_template_str( "specificationVersion: jobtemplate-2023-09\n" "name: T\n" From 124d3f4a1fe5a1e669bb0a5d4030213b122f55d3 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:18:04 +0000 Subject: [PATCH 05/21] fix!: Pin the remaining path mapping, session and YAML changes from the bump A third review of the bump found more visible changes: - The 32,768-byte path limit (openjd-rs#472) has three outcomes, not one. PathMappingRule.apply returns no match for a longer path, the apply_path_mapping expression function raises ExpressionError, and a session errors on a longer PATH or LIST[PATH] value. The specs said only "matches no rule". Correct them, and pin apply_path_mapping. - Between a POSIX and a WINDOWS rule with the same number of names, the WINDOWS rule applies in any order (openjd-rs#472). Pin and document it. - The session working directory is named with 8 random characters and no part of the session ID (openjd-rs#444), as the pure-Python reference names it. Pin and document it. - A YAML 1.2 Core tag on a scalar, such as "!!float 1.5", decodes with serde-saphyr 2.1. With 1.2 it failed. Pin it. Not pinned here because this repo's tests can't reach them, or they only remove a crash: openjd-sessions 0.9.0 maps every LIST[PATH] value when the session has a base symbol table (openjd-rs#483); a malformed openjd_unset_env line no longer also unsets a variable named after the parse error, and redacting a value that starts with a multibyte character no longer panics (openjd-rs#515); with_number no longer overflows on i64::MIN (openjd-rs#515); and CHUNK[INT] chunking no longer overflows or lists every value for huge ranges (openjd-rs#473). BREAKING CHANGE: apply_path_mapping raises ExpressionError for a path longer than 32,768 bytes, and a session errors on a PATH or LIST[PATH] value that long. A WINDOWS path mapping rule applies over a POSIX rule with the same number of names. Session.working_directory no longer contains the session ID. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- specs/python-expr-interface.md | 16 ++++--- specs/python-sessions-interface.md | 11 ++++- test/openjd/expr/test_path_mapping.py | 43 ++++++++++++++++++- test/openjd/model_v1/test_parse.py | 23 ++++++++++ .../sessions/test_path_mapping_rules.py | 16 +++++++ 5 files changed, 100 insertions(+), 9 deletions(-) diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index 170ff626..fec03d5d 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -372,12 +372,15 @@ such as a POSIX or WINDOWS rule whose `source_path` is not absolute; the message names the rule's index, for example `path mapping rule 1: source_path 'rel' must be an absolute POSIX path, starting with '/'`. When several rules match a path, the one whose source path has the -most names applies, wherever it is in the list, and between rules for -the same location the last one given applies (How Jobs Are Run, "Path -Mapping"). The order of `rules` matters only for that tie. +most names applies, wherever it is in the list. Between a POSIX rule +and a WINDOWS rule with the same number of names (both can match a +path such as `//srv/share/x`), the WINDOWS rule applies. Between rules +for the same location, the last one given applies (How Jobs Are Run, +"Path Mapping"). The order of `rules` matters only for that last tie. A `source_path` or `destination_path` longer than 32,768 bytes makes a -rule invalid, a path longer than that matches no rule, and the rules -together are limited to about 30 MiB of lookup structure. +rule invalid, and the rules together are limited to about 30 MiB of +lookup structure. `apply_path_mapping` raises `ExpressionError` for a +path longer than 32,768 bytes. ```python # Inspecting a profile @@ -655,7 +658,8 @@ matched, result = rule.apply(path="/other/path") # source_path or destination_path is longer than 32,768 bytes, can be # constructed, but apply() raises ValueError for it: # "source_path 'rel' must be an absolute POSIX path, starting with '/'" -# A path longer than 32,768 bytes matches no rule. +# A path longer than 32,768 bytes matches no rule here (apply_path_mapping +# raises for it instead). # Serialization d = rule.to_dict() diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index 901c4aa7..a9c1491a 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -182,9 +182,16 @@ longer than 32,768 bytes (How Jobs Are Run, "Path Mapping"). A set with an invalid rule raises `SessionError`, and `extend_path_mapping_rules` leaves the existing rules unchanged. When several rules match a path, the one whose source path has the most -names applies, wherever it is in the list. Between rules for the same -location the last one applies, so a rule added by +names applies, wherever it is in the list, and a WINDOWS rule applies +over a POSIX rule with the same number of names. Between rules for the +same location the last one applies, so a rule added by `extend_path_mapping_rules` replaces an existing rule for its location. +A PATH or LIST[PATH] value longer than 32,768 bytes is an error when +the session resolves it. + +`working_directory` is a directory under the session root whose name +is 8 random characters from `[a-z0-9_]`, with no part of the session +ID in it, as the pure-Python reference names it. ### SessionUser (stays in Python, passed to Rust) diff --git a/test/openjd/expr/test_path_mapping.py b/test/openjd/expr/test_path_mapping.py index 252f0c1a..a0c789a2 100644 --- a/test/openjd/expr/test_path_mapping.py +++ b/test/openjd/expr/test_path_mapping.py @@ -7,7 +7,7 @@ import pytest -from openjd.expr import PathMappingRule, PathFormat, ExprProfile, HostContext +from openjd.expr import ExpressionError, PathMappingRule, PathFormat, ExprProfile, HostContext class TestPathMappingRuleFromPosix: @@ -457,6 +457,19 @@ def test_a_longer_path_matches_no_rule(self) -> None: path = "/mnt/" + "a" * (32 * 1024 - len("/mnt/") + 1) assert self._RULE.apply(path=path) == (False, path) + def test_apply_path_mapping_rejects_a_longer_path(self) -> None: + """The ``apply_path_mapping`` expression function raises rather than + returning the path unmapped.""" + from openjd.expr import parse_expression + + profile = ExprProfile().with_host_context(HostContext.with_rules([self._RULE])) + path = "/mnt/" + "a" * (32 * 1024 - len("/mnt/") + 1) + with pytest.raises(ExpressionError) as exc_info: + parse_expression(f"apply_path_mapping('{path}')").evaluate(profile=profile) + assert str(exc_info.value).split("\n")[0] == ( + "apply_path_mapping: the path is 32769 bytes long, more than the limit of 32768 bytes" + ) + def test_a_longer_destination_path_is_invalid(self) -> None: rule = PathMappingRule( source_path_format=PathFormat.POSIX, @@ -708,6 +721,34 @@ def test_the_rule_with_the_most_names_wins_in_any_order(self, tmp_path, reverse) parsed = parse_expression("apply_path_mapping('/mnt/shared/file.exr')") assert parsed.evaluate(profile=profile).item() == long_dest + os.sep + "file.exr" + @pytest.mark.parametrize("reverse", [False, True], ids=["WINDOWS first", "POSIX first"]) + def test_windows_wins_a_tie_with_posix_in_any_order(self, reverse) -> None: + """A POSIX rule and a WINDOWS rule with the same number of names can + both match a path such as ``//srv/share/x``, and the WINDOWS rule + applies wherever it is in the list (How Jobs Are Run, "Applying Path + Mapping Rules within a Job Template", item 3). openjd-expr 0.14.0 + (openjd-rs#472); before, the first one applied.""" + from openjd.expr import parse_expression + + rules = [ + PathMappingRule( + source_path_format=PathFormat.WINDOWS, + source_path="\\\\srv\\share", + destination_path="/win", + ), + PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path="//srv/share", + destination_path="/posix", + ), + ] + if reverse: + rules.reverse() + profile = ExprProfile().with_host_context(HostContext.with_rules(rules)) + parsed = parse_expression("apply_path_mapping('//srv/share/x')") + result = parsed.evaluate(profile=profile, path_format=PathFormat.POSIX) + assert result.item() == "/win/x" + def test_the_last_rule_for_the_same_location_wins(self, tmp_path) -> None: """Rules whose source paths have the same components name the same location, and the last one given applies (How Jobs Are Run, "Applying diff --git a/test/openjd/model_v1/test_parse.py b/test/openjd/model_v1/test_parse.py index 27955922..f0a1311e 100644 --- a/test/openjd/model_v1/test_parse.py +++ b/test/openjd/model_v1/test_parse.py @@ -389,6 +389,29 @@ def test_mixed_case_bool_and_null_spellings_are_strings(self, scalar: str) -> No assert t.parameter_definitions is not None assert t.parameter_definitions[0].default == scalar + @pytest.mark.parametrize( + "ptype, scalar, expected", + [ + pytest.param("FLOAT", "!!float 1.5", 1.5, id="float"), + pytest.param("INT", "!!int 3", 3, id="int"), + ], + ) + def test_a_core_tagged_scalar_decodes(self, ptype: str, scalar: str, expected: Any) -> None: + """With serde-saphyr 2.1 a YAML 1.2 Core tag on a scalar is read as its + type. Before, with serde-saphyr 1.2, the decode failed with + ``cannot deserialize tagged scalar into string``.""" + t = decode_job_template_str( + "specificationVersion: jobtemplate-2023-09\n" + "name: T\n" + "parameterDefinitions:\n" + f"- {{name: P, type: {ptype}, default: {scalar}}}\n" + "steps:\n" + "- name: S\n" + " script: {actions: {onRun: {command: echo}}}\n" + ) + assert t.parameter_definitions is not None + assert t.parameter_definitions[0].default == expected + class TestDecodeEnvironmentTemplateStr: """``decode_environment_template_str`` wrapper: parses YAML or diff --git a/test/openjd/sessions/test_path_mapping_rules.py b/test/openjd/sessions/test_path_mapping_rules.py index 0fdb6ce9..b220c3f6 100644 --- a/test/openjd/sessions/test_path_mapping_rules.py +++ b/test/openjd/sessions/test_path_mapping_rules.py @@ -12,6 +12,7 @@ from __future__ import annotations +import os import time import pytest @@ -60,3 +61,18 @@ def test_extend_rejects_a_relative_source_path() -> None: assert str(exc_info.value) == _MESSAGE finally: session.cleanup() + + +def test_the_working_directory_is_named_with_8_random_characters() -> None: + """Since openjd-sessions 0.9.0 (openjd-rs#444) the session working directory + is named with exactly 8 characters from ``[a-z0-9_]`` and no part of the + session ID, as the pure-Python reference's ``mkdtemp(prefix="")`` names it. + Before, the name was the session ID followed by 32 hex digits.""" + import re + + session = Session(session_id="my-session-id", job_parameter_values={}) + try: + name = os.path.basename(session.working_directory) + assert re.fullmatch(r"[a-z0-9_]{8}", name), name + finally: + session.cleanup() From 9ddb1e2c6148fe4c794a33fb98925f38d2796d42 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:24:19 +0000 Subject: [PATCH 06/21] test!: Pin the component matching changes in path mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fourth review of the bump found that the PathMapper rewrite (openjd-rs#472) also changed how rules match components, as How Jobs Are Run, "Path Mapping" item 2 requires: - A UNC share is one component and must match as a whole, so a rooted path such as \srv\share\x no longer matches a \\srv\share rule. - WINDOWS components compare the way Windows compares file names, non-ASCII letters included, so C:\Ünï matches C:\üNÏ. The pure-Python reference in openjd-sessions folds only ASCII letters. - "." and empty components are ignored when matching and left out of the mapped path, so /mnt/./a/b matches a /mnt/a rule, and /mnt/a/./b//c maps to /d/b/c. Pin each with a test and describe them in the expr spec, with a pointer from the sessions spec. BREAKING CHANGE: a rooted WINDOWS path no longer matches a UNC share rule, WINDOWS rules match non-ASCII letters case-insensitively, and "." and empty components are ignored when matching and dropped from the mapped path. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- specs/python-expr-interface.md | 6 +++ specs/python-sessions-interface.md | 7 ++- test/openjd/expr/test_path_mapping.py | 72 +++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index fec03d5d..76693ff2 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -377,6 +377,12 @@ and a WINDOWS rule with the same number of names (both can match a path such as `//srv/share/x`), the WINDOWS rule applies. Between rules for the same location, the last one given applies (How Jobs Are Run, "Path Mapping"). The order of `rules` matters only for that last tie. +Paths are compared component by component, as How Jobs Are Run, "Path +Mapping" item 2 says: a drive or UNC share must match as a whole, +WINDOWS components compare the way Windows compares file names +(non-ASCII letters included), and `.` and empty components are ignored +and left out of the mapped path. The pure-Python reference in +openjd-sessions folds only ASCII letters. A `source_path` or `destination_path` longer than 32,768 bytes makes a rule invalid, and the rules together are limited to about 30 MiB of lookup structure. `apply_path_mapping` raises `ExpressionError` for a diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index a9c1491a..55c14a48 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -186,8 +186,11 @@ names applies, wherever it is in the list, and a WINDOWS rule applies over a POSIX rule with the same number of names. Between rules for the same location the last one applies, so a rule added by `extend_path_mapping_rules` replaces an existing rule for its location. -A PATH or LIST[PATH] value longer than 32,768 bytes is an error when -the session resolves it. +Rules match paths component by component, as described for +`HostContext.with_rules` in +[python-expr-interface.md](./python-expr-interface.md). A PATH or +LIST[PATH] value longer than 32,768 bytes is an error when the session +resolves it. `working_directory` is a directory under the session root whose name is 8 random characters from `[a-z0-9_]`, with no part of the session diff --git a/test/openjd/expr/test_path_mapping.py b/test/openjd/expr/test_path_mapping.py index a0c789a2..e1239ca2 100644 --- a/test/openjd/expr/test_path_mapping.py +++ b/test/openjd/expr/test_path_mapping.py @@ -415,6 +415,78 @@ def test_windows_trailing_forward_slash(self, tmp_path) -> None: assert result.endswith("/") or result.endswith(os.sep) or result.endswith("\\") +class TestComponentMatching: + """Since openjd-expr 0.14.0 (openjd-rs#472) rules match whole components + the way How Jobs Are Run, "Applying Path Mapping Rules within a Job + Template", item 2 says: a UNC share is one component and must match as a + whole, WINDOWS components compare the way Windows compares file names + (non-ASCII letters too), and ``.`` and empty components are ignored and + left out of the mapped path. Before, a rooted path such as ``\\srv\\share`` + matched a UNC rule, ``C:\\Ünï`` did not match ``C:\\üNÏ``, a ``.`` before + the end of the source path prevented a match, and ``.`` and empty + components after it were kept. The pure-Python reference agrees on every + row except the non-ASCII one, where it folds ASCII letters only.""" + + @pytest.mark.parametrize( + "source_path_format, source_path, path, expected", + [ + pytest.param( + PathFormat.WINDOWS, + "\\\\srv\\share", + "\\\\SRV\\SHARE\\x", + (True, "/d/x"), + id="UNC share, case-insensitive", + ), + pytest.param( + PathFormat.WINDOWS, + "\\\\srv\\share", + "\\srv\\share\\x", + (False, "\\srv\\share\\x"), + id="a rooted path is not a UNC share", + ), + pytest.param( + PathFormat.WINDOWS, + "C:\\Ünï", + "C:\\üNÏ\\f", + (True, "/d/f"), + id="non-ASCII letters compare case-insensitively", + ), + pytest.param( + PathFormat.WINDOWS, + "C:\\proj", + "C:\\proj\\.\\f\\", + (True, "/d/f/"), + id="WINDOWS dot component dropped", + ), + pytest.param( + PathFormat.POSIX, + "/mnt/a", + "/mnt/./a/b", + (True, "/d/b"), + id="dot component inside the source path", + ), + pytest.param( + PathFormat.POSIX, + "/mnt/a", + "/mnt/a/./b//c", + (True, "/d/b/c"), + id="dot and empty components dropped", + ), + ], + ) + def test_apply( + self, + source_path_format: PathFormat, + source_path: str, + path: str, + expected: tuple[bool, str], + ) -> None: + rule = PathMappingRule( + source_path_format=source_path_format, source_path=source_path, destination_path="/d" + ) + assert rule.apply(path=path, output_format=PathFormat.POSIX) == expected + + class TestDestinationEndingInASeparator: """Since openjd-expr 0.14.0 (openjd-rs#483) no separator is added after a destination that already ends with one. Before, ``/dst/`` gave From 4ba50ef2e4b3962ecf93a2cd2a1f8e3e6cefb030 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:25:31 +0000 Subject: [PATCH 07/21] test: Pin the range expression domain, element cap and nesting changes Reading the upstream commits behind the bump turned up three more visible openjd-expr changes: - RangeExpr values can be any 64-bit integer except the most negative one (openjd-rs#469). Before, magnitudes of 2**62 and up raised an overflow error. - Parsing accepts up to 50,000 comma-separated elements (openjd-rs#470). Before, the cap was 10,000. - A range_expr inside two list levels, [[range_expr('1-3')]], is two levels deep (openjd-rs#462). Before, it raised "Lists may be nested at most 2 levels deep". Each is a relaxation, so nothing that worked before now fails. Pin them and describe the first two in the expr spec. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- specs/python-expr-interface.md | 4 +++- test/openjd/expr/test_lists.py | 8 ++++++++ test/openjd/expr/test_range_expr.py | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index 76693ff2..89e3b83f 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -765,7 +765,9 @@ A `RangeExpr` is kept in the canonical form of Template Schemas `1-9:2` and `RangeExpr("1-2").ranges()` is two single-value ranges. Two `RangeExpr`s are equal exactly when they have the same values. Every constructor raises `RangeExprError` when the canonical form would -have more than 50,000 runs. The pure-Python +have more than 50,000 runs, and parsing accepts at most 50,000 +comma-separated elements. Values can be any 64-bit integer except the +most negative one. The pure-Python `openjd.model._range_expr.IntRangeExpr` follows the input's elements instead. diff --git a/test/openjd/expr/test_lists.py b/test/openjd/expr/test_lists.py index 15c8b632..6391da15 100644 --- a/test/openjd/expr/test_lists.py +++ b/test/openjd/expr/test_lists.py @@ -124,6 +124,14 @@ def test_three_level_nesting_in_comprehension_fails(self) -> None: with pytest.raises(ExpressionError, match="nested at most 2 levels"): evaluate_expression("[[[x]] for x in [1, 2]]") + def test_a_range_expr_in_two_levels_is_not_a_third(self) -> None: + """A ``range_expr`` element is not a list level, so ``[[r]]`` is two + levels deep (Expression Language §1.2.1). openjd-expr 0.13.0 + (openjd-rs#462); before, it raised "Lists may be nested at most 2 + levels deep".""" + result = evaluate_expression("[[range_expr('1-3')]]") + assert result.type == ExprType("list[list[range_expr]]") + class TestListElementCoercion: """Tests for list element coercion when target type has single list type (RFC 0005).""" diff --git a/test/openjd/expr/test_range_expr.py b/test/openjd/expr/test_range_expr.py index aaeb1d79..f039abe1 100644 --- a/test/openjd/expr/test_range_expr.py +++ b/test/openjd/expr/test_range_expr.py @@ -220,6 +220,21 @@ def test_equality_is_by_value(self) -> None: assert a == b assert hash(a) == hash(b) + def test_values_span_the_64_bit_range(self) -> None: + """openjd-expr 0.13.0 (openjd-rs#469) widened the value domain from + magnitudes below 2**62 to every 64-bit integer except the most + negative one. Before, both of these raised an overflow error.""" + assert str(RangeExpr("-9223372036854775807,9223372036854775807")) == ( + "-9223372036854775807,9223372036854775807" + ) + assert str(RangeExpr("4611686018427387904")) == "4611686018427387904" + + def test_parsing_accepts_more_than_10000_elements(self) -> None: + """openjd-expr 0.14.0 (openjd-rs#470) raised the element cap from + 10,000 to 50,000. Before, 12,000 elements raised ``RangeExprError``.""" + r = RangeExpr(",".join(str(i * 3) for i in range(12_000))) + assert r.ranges() == [IntRange(0, 35_997, 3)] + def test_from_list_caps_the_canonical_runs(self) -> None: """openjd-expr 0.14.0 (openjd-rs#470) caps every constructor at 50,000 runs of the canonical form. 25,000 pairs are 50,000 runs, and one more From b997af42f71e6436e23efa58c3c6a786eb9e690c Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:33:20 +0000 Subject: [PATCH 08/21] test: Correct two references in the bump's tests and spec The expr spec, and the body of the commit that pinned component matching, cited How Jobs Are Run, "Path Mapping" item 2. The rule is item 2 of its "Applying Path Mapping Rules within a Job Template" subsection. The working directory test said the new name came with openjd-sessions 0.9.0; openjd-rs#444 first shipped in 0.8.2, which this repo skipped. Move that test to its own file, since it is not about path mapping rules. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- specs/python-expr-interface.md | 5 ++-- .../sessions/test_path_mapping_rules.py | 16 ------------- .../openjd/sessions/test_working_directory.py | 24 +++++++++++++++++++ 3 files changed, 27 insertions(+), 18 deletions(-) create mode 100644 test/openjd/sessions/test_working_directory.py diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index 89e3b83f..f98e0a7d 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -377,8 +377,9 @@ and a WINDOWS rule with the same number of names (both can match a path such as `//srv/share/x`), the WINDOWS rule applies. Between rules for the same location, the last one given applies (How Jobs Are Run, "Path Mapping"). The order of `rules` matters only for that last tie. -Paths are compared component by component, as How Jobs Are Run, "Path -Mapping" item 2 says: a drive or UNC share must match as a whole, +Paths are compared component by component, as How Jobs Are Run, +"Applying Path Mapping Rules within a Job Template", item 2 says: a +drive or UNC share must match as a whole, WINDOWS components compare the way Windows compares file names (non-ASCII letters included), and `.` and empty components are ignored and left out of the mapped path. The pure-Python reference in diff --git a/test/openjd/sessions/test_path_mapping_rules.py b/test/openjd/sessions/test_path_mapping_rules.py index b220c3f6..0fdb6ce9 100644 --- a/test/openjd/sessions/test_path_mapping_rules.py +++ b/test/openjd/sessions/test_path_mapping_rules.py @@ -12,7 +12,6 @@ from __future__ import annotations -import os import time import pytest @@ -61,18 +60,3 @@ def test_extend_rejects_a_relative_source_path() -> None: assert str(exc_info.value) == _MESSAGE finally: session.cleanup() - - -def test_the_working_directory_is_named_with_8_random_characters() -> None: - """Since openjd-sessions 0.9.0 (openjd-rs#444) the session working directory - is named with exactly 8 characters from ``[a-z0-9_]`` and no part of the - session ID, as the pure-Python reference's ``mkdtemp(prefix="")`` names it. - Before, the name was the session ID followed by 32 hex digits.""" - import re - - session = Session(session_id="my-session-id", job_parameter_values={}) - try: - name = os.path.basename(session.working_directory) - assert re.fullmatch(r"[a-z0-9_]{8}", name), name - finally: - session.cleanup() diff --git a/test/openjd/sessions/test_working_directory.py b/test/openjd/sessions/test_working_directory.py new file mode 100644 index 00000000..8dcafae5 --- /dev/null +++ b/test/openjd/sessions/test_working_directory.py @@ -0,0 +1,24 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""How ``Session`` names its working directory.""" + +from __future__ import annotations + +import os +import re + +from openjd._openjd_rs import Session + + +def test_the_working_directory_is_named_with_8_random_characters() -> None: + """Since openjd-sessions 0.8.2 (openjd-rs#444) the session working directory + is named with exactly 8 characters from ``[a-z0-9_]`` and no part of the + session ID, as the pure-Python reference's ``mkdtemp(prefix="")`` names it. + Before, the name was the session ID followed by 32 hex digits.""" + session = Session(session_id="my-session-id", job_parameter_values={}) + try: + name = os.path.basename(session.working_directory) + assert re.fullmatch(r"[a-z0-9_]{8}", name), name + finally: + session.cleanup() From 683d79d3fa2f9c441a40220f5f269311570d3c0e Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:09:31 +0000 Subject: [PATCH 09/21] fix!: Bump the bindings to openjd-expr 0.15, openjd-model 0.16 and openjd-sessions 0.10 Move rust-bindings from openjd-expr 0.14.0, openjd-model 0.15.0 and openjd-sessions 0.9.0 to 0.15.0, 0.16.0 and 0.10.0. The bindings compile unchanged. Cargo.lock swaps RustPython's rustpython-ruff_python_* 0.16.10 for Astral's ruff_python_* 0.0.21, drops the icu_* crates, and moves phf to 0.14. The main reason is openjd-rs#531: openjd-expr now parses with Astral's ruff_python_parser, so an invalid expression parses in time roughly linear in its length. With 0.14.0, decode_job_template took 2.6 s in a release build (39 s in a debug build) on the 101-byte "[i f(" * 20 + "x", and the time doubled per unclosed bracket. The pure-Python model is affected too, since it evaluates EXPR expressions with the same parser. A new test_parser_work_bound.py runs the openjd-rs#489 reproducers through parse_expression, FormatString and both decode_job_template functions with a 1 s bound; each takes a few milliseconds now. Tests changed for intended upstream changes (openjd-rs#531): - Syntax errors use Ruff's wording: "unexpected EOF while parsing" rather than "'(' was never closed", and "missing closing quote in string literal" rather than "unterminated string literal". Three tests in TestSyntaxErrorCarets. - A multi-line expression is parsed as written rather than wrapped in parentheses, so a binary operator's caret sits on the operator (~~^~~~~) instead of one character to its right. Four tests in TestMultiLineExpressions. New tests pin the other visible changes: - More of Ruff's wording, including a two-line expression's error on its second line (openjd-rs#531). - Keyword placeholders no longer collide with an NFKC-normalized name or fall back to an unchecked one (openjd-rs#534). - A FLOAT given as a number is written as Python's repr writes it: 2 is "2.0" and 1e20 is "1e+20" (openjd-rs#539). - A native list of numeric strings is accepted for LIST[INT], LIST[FLOAT] and LIST[LIST[INT]], and a LIST[FLOAT] default string item keeps its spelling (openjd-rs#538). - White space around a numeric string value is ignored (openjd-rs#530). - A whole-field FLOAT range expression keeps each float's spelling (openjd-rs#540). - Sessions: a task or environment that fails before it starts fails the action (openjd-rs#532); a failed action keeps the session ending-only after a successful exit_environment(keep_session_running= True) (openjd-rs#524); onExit runs with the enter-time symbol table when the exit-time one can't be built (openjd-rs#536). A new strict xfail in test/openjd/sessions/test_known_gaps.py records that a run_task the crate refuses, such as on an ending-only session, returns None and leaves the previous action_status, where the pure-Python reference raises. The binding still discards the crate's Result for its non-blocking actions; openjd-rs#524 makes that case reachable. No strict xfail XPASSes. The three interface specs describe the changes. No public signature changed. BREAKING CHANGE: syntax error messages use Ruff's wording, and a multi-line expression's caret moves one character left. A FLOAT given as a number is written as Python's repr writes it. A _v1 Session whose action failed stays READY_ENDING after exit_environment( keep_session_running=True), and a run_task or enter_environment that fails before it starts now fails the action and makes the session ending-only. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- Cargo.lock | 510 +++++++----------- rust-bindings/Cargo.toml | 6 +- specs/python-expr-interface.md | 10 +- specs/python-model-interface.md | 12 + specs/python-sessions-interface.md | 17 + test/openjd/expr/test_error_formatting.py | 59 +- test/openjd/expr/test_parser_work_bound.py | 129 +++++ test/openjd/expr/test_parsing.py | 23 + test/openjd/model_v1/test_create_job.py | 102 ++++ .../model_v1/test_step_param_space_iter.py | 45 ++ test/openjd/sessions/test_failed_actions.py | 153 ++++++ test/openjd/sessions/test_known_gaps.py | 60 +++ 12 files changed, 805 insertions(+), 321 deletions(-) create mode 100644 test/openjd/expr/test_parser_work_bound.py create mode 100644 test/openjd/sessions/test_failed_actions.py create mode 100644 test/openjd/sessions/test_known_gaps.py diff --git a/Cargo.lock b/Cargo.lock index 14622577..90191e29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + [[package]] name = "char_str" version = "0.0.4" @@ -223,6 +234,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -235,17 +255,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -[[package]] -name = "displaydoc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", -] - [[package]] name = "drop_bomb" version = "0.1.5" @@ -303,6 +312,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.11" @@ -453,69 +468,6 @@ dependencies = [ "cc", ] -[[package]] -name = "icu_collections" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_properties" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" - -[[package]] -name = "icu_provider" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - [[package]] name = "indexmap" version = "2.14.1" @@ -611,12 +563,6 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" -[[package]] -name = "litemap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" - [[package]] name = "log" version = "0.4.34" @@ -791,26 +737,28 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openjd-expr" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2aa8e58546ad6ee7baf11ad8a547694f0d49863e635e7f1a7ac000e6b79123b" +checksum = "379859f4b71a91967f6600d2204691c62f04e27b6cdd9f3ea2e4f6c22ff691b3" dependencies = [ "regex", "regex-syntax", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_parser", - "rustpython-ruff_text_size", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", + "ryu", "serde", "serde_json", "shlex", "thiserror", + "unicode-normalization", ] [[package]] name = "openjd-model" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144e06e705f61da6e83b64ebf0c237a46199de28e8deadc7fdd26acb44ccf139" +checksum = "4a24e459b033edce4ca749dcb9600dae160e0d6cd50f920b02ac867367c5631a" dependencies = [ "indexmap", "openjd-expr", @@ -840,9 +788,9 @@ dependencies = [ [[package]] name = "openjd-sessions" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32109f68122a780b053c782be55d365ea7486a025ae3e4dad22e83065808698" +checksum = "78edbce036740c750ec32f149d17a985c42ff32ccfc4da66e0530c1ff8cdcd5a" dependencies = [ "bitflags", "futures-util", @@ -887,7 +835,16 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_shared 0.14.0", ] [[package]] @@ -896,8 +853,18 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_codegen" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" +dependencies = [ + "phf_generator 0.14.0", + "phf_shared 0.14.0", ] [[package]] @@ -906,8 +873,18 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", - "rand", + "phf_shared 0.11.3", + "rand 0.8.8", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared 0.14.0", ] [[package]] @@ -919,6 +896,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -940,15 +926,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "potential_utf" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" -dependencies = [ - "zerovec", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -1147,7 +1124,17 @@ checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "rand_core 0.10.1", ] [[package]] @@ -1157,7 +1144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -1169,6 +1156,12 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rawpointer" version = "0.2.1" @@ -1204,6 +1197,106 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ruff_python_ast" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8f77693e5f11e7d574c84a8dd63190142ae683749f5ef019bafeffb952fd19" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags", + "char_str", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "thin-vec", + "thiserror", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c567873f6a0e094d3f72b37862a8420a6eb4b3fc2950b4975e5a61b46aab3acc" +dependencies = [ + "bitflags", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash 2.1.3", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78209bfee4543e650edffee040849c5ca828a05d60c7d3b8b53bb63d2dd49a70" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2159c6129bf08ecde949e8aca0edc5411fbcbaaecdbd401fd75385f5a74e95" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb9ffcbeaefe5f3a7bef2c80dbf591a0fa1f9ec22c5ce7bf4f4c2397104e6496" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc534053fd582ce4b3431d94f6e9f398bb214eb9baf497b06b5c8d6446f379d" +dependencies = [ + "phf 0.14.0", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b540b2b312bde02dbfd5261013e06dda9d3ecd33cb6e62f962b019903ae56c" +dependencies = [ + "phf_codegen 0.14.0", + "rand 0.10.3", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1241,8 +1334,8 @@ dependencies = [ "log", "num-bigint", "num-traits", - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", "rustc-hash 1.1.0", "rustpython-ast", "rustpython-parser-core", @@ -1273,87 +1366,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "rustpython-ruff_python_ast" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7ffeb85f9a68c461d538839c79548584768df08f2d68cb1ee693be697e0355" -dependencies = [ - "aho-corasick", - "arrayvec", - "bitflags", - "char_str", - "compact_str", - "get-size2", - "is-macro", - "memchr", - "rustc-hash 2.1.3", - "rustpython-ruff_python_trivia", - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", - "thin-vec", - "thiserror", -] - -[[package]] -name = "rustpython-ruff_python_parser" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c9729e35cac02839c7a67aaccfea3eba32032f7aeacca2e82ad79e8d7b7fd95" -dependencies = [ - "bitflags", - "bstr", - "drop_bomb", - "get-size2", - "hashbrown", - "icu_properties", - "memchr", - "rustc-hash 2.1.3", - "rustpython-ruff_python_ast", - "rustpython-ruff_python_trivia", - "rustpython-ruff_text_size", - "stacker", - "static_assertions", - "thin-vec", - "unicode-ident", - "unicode-normalization", - "unicode_names2", -] - -[[package]] -name = "rustpython-ruff_python_trivia" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764f43ea060f888164fbd360b0e1a36f2a35f5c03def85407bab7039fc62a645" -dependencies = [ - "itertools 0.15.0", - "memchr", - "rustc-hash 2.1.3", - "rustpython-ruff_source_file", - "rustpython-ruff_text_size", - "unicode-ident", - "unicode-normalization", -] - -[[package]] -name = "rustpython-ruff_source_file" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe26430c3f2eadb40524b6df7ac9a3970dd2b58f921b01ed75df0121e332fad" -dependencies = [ - "memchr", - "rustpython-ruff_text_size", -] - -[[package]] -name = "rustpython-ruff_text_size" -version = "0.16.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5377a04f2855030600b93a05754242950ea9b070507ef9aef67965a52eeb09" -dependencies = [ - "get-size2", -] - [[package]] name = "rustversion" version = "1.0.23" @@ -1478,12 +1490,6 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - [[package]] name = "stacker" version = "0.1.25" @@ -1604,17 +1610,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "synstructure" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", -] - [[package]] name = "target-lexicon" version = "0.13.5" @@ -1675,16 +1670,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "tinystr" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" -dependencies = [ - "displaydoc", - "zerovec", -] - [[package]] name = "tinyvec" version = "1.12.0" @@ -1863,7 +1848,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" dependencies = [ - "phf", + "phf 0.11.3", "unicode_names2_generator", ] @@ -1875,16 +1860,10 @@ checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" dependencies = [ "getopts", "log", - "phf_codegen", - "rand", + "phf_codegen 0.11.3", + "rand 0.8.8", ] -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "uuid" version = "1.26.0" @@ -2108,35 +2087,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" -[[package]] -name = "writeable" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", - "synstructure", -] - [[package]] name = "zerocopy" version = "0.8.56" @@ -2157,60 +2107,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", - "synstructure", -] - -[[package]] -name = "zerotrie" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", -] - [[package]] name = "zmij" version = "1.0.23" diff --git a/rust-bindings/Cargo.toml b/rust-bindings/Cargo.toml index 5157a7fc..7136b91d 100644 --- a/rust-bindings/Cargo.toml +++ b/rust-bindings/Cargo.toml @@ -12,9 +12,9 @@ name = "_openjd_rs" crate-type = ["cdylib", "rlib"] [dependencies] -openjd-expr = "0.14.0" -openjd-model = "0.15.0" -openjd-sessions = "0.9.0" +openjd-expr = "0.15.0" +openjd-model = "0.16.0" +openjd-sessions = "0.10.0" tokio = { version = "1", features = ["rt-multi-thread"] } uuid = { version = "1", features = ["v4"] } serde_json = "1" diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index f98e0a7d..c7e9e3e8 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -957,7 +957,7 @@ from openjd.expr import ExpressionError, RangeExprError, evaluate_expression try: evaluate_expression("1 +") except ExpressionError as e: - str(e) # "Syntax error: ...\n 1 +\n ^~~" + str(e) # "Syntax error: Expected an expression\n 1 +\n ^~~" # Undefined variable try: @@ -972,6 +972,14 @@ except RangeExprError as e: str(e) # "Expected integer in 'not-a-range'" ``` +Syntax errors use the wording of Ruff's parser (`Expected an +expression`, `unexpected EOF while parsing`), which openjd-expr uses +since 0.15.0 (openjd-rs#531). Parsing an invalid expression takes time +roughly linear in its length, whatever its bracket nesting. A +multi-line expression is parsed as written, so a caret points into the +text the author wrote. + + | Exception | Base | |---|---| | `ExpressionError` | `ValueError` | diff --git a/specs/python-model-interface.md b/specs/python-model-interface.md index 701c3d44..dc7b83df 100644 --- a/specs/python-model-interface.md +++ b/specs/python-model-interface.md @@ -287,6 +287,18 @@ value is joined to `current_working_dir`. Joined values are lexically normalized; an absolute submitted value is returned as written. A default's constraints are checked against the joined value. URI values are refused. Violations raise `DecodeValidationError` (openjd-rs#421). +A FLOAT given as a number, as a default, a `LIST[FLOAT]` item or a +value, is written the way Python's `repr(float)` writes it, with zero +always `0.0`: `default: 2` gives `"2.0"` and `1e20` gives `"1e+20"`. A +FLOAT given as a string keeps its spelling (Template Schemas §7.5, +openjd-rs#539). White space around a number in a string value is +ignored, so `" 7 "` is the INT `7` (openjd-rs#530). A `LIST[INT]`, +`LIST[FLOAT]` or `LIST[LIST[INT]]` value given as a Python list of +numeric strings is coerced item by item, the same as the JSON text of +that list, and an item error names its position (`item[1]: ...`); a +`LIST[FLOAT]` default item written as a string keeps its spelling +(openjd-rs#538). + ```python from openjd.model._v1 import decode_job_template, preprocess_job_parameters from pathlib import Path diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index 55c14a48..9c4654d6 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -196,6 +196,23 @@ resolves it. is 8 random characters from `[a-z0-9_]`, with no part of the session ID in it, as the pure-Python reference names it. +An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session +ending-only (`READY_ENDING`) for good: `exit_environment` and `cleanup` +still work, and a successful +`exit_environment(keep_session_running=True)` leaves it `READY_ENDING` +(openjd-rs#524). When `run_task` or `enter_environment` fails before its +subprocess starts, for example because a parameter value can't be +coerced, the action is `FAILED` with the error's message, as in the +pure-Python reference (openjd-rs#532); an environment that fails this +way still counts as entered. When `exit_environment` can't build its +symbol table from `resolved_symtab`, it runs `onExit` with the table the +environment was entered with (openjd-rs#536). + +Known gap: a call the session refuses, such as `run_task` on a +`READY_ENDING` session, returns `None` without running anything, and +`action_status` still describes the previous action. The pure-Python +reference raises. `clear_ending_only()` is not bound yet. + ### SessionUser (stays in Python, passed to Rust) The `SessionUser` types involve OS-specific credential validation (sudo diff --git a/test/openjd/expr/test_error_formatting.py b/test/openjd/expr/test_error_formatting.py index 8268fc56..b4fb5170 100644 --- a/test/openjd/expr/test_error_formatting.py +++ b/test/openjd/expr/test_error_formatting.py @@ -302,15 +302,21 @@ def test_leading_whitespace_error_in_middle(self): class TestSyntaxErrorCarets: - """Test that syntax errors include caret pointers.""" + """Test that syntax errors include caret pointers. + + Since openjd-expr 0.15.0 (openjd-rs#531) expressions are parsed with + Astral's ruff_python_parser, so syntax errors use Ruff's wording + ("unexpected EOF while parsing"). From openjd-expr 0.11.0 (openjd-rs#429) + until then, RustPython's fork of the parser gave CPython's wording + ("'(' was never closed"). The caret is unchanged. + """ def test_unclosed_paren(self): """Unclosed parenthesis shows caret spanning the expression.""" with pytest.raises(ExpressionError) as exc_info: evaluate_expression("(1 + 2", values=SymbolTable()) - # CPython's wording since ruff 0.16.10 (openjd-rs#429). expected = [ - "Syntax error: '(' was never closed\n", + "Syntax error: unexpected EOF while parsing\n", " (1 + 2\n", " ^~~~~~", ] @@ -321,7 +327,7 @@ def test_unclosed_bracket(self): with pytest.raises(ExpressionError) as exc_info: evaluate_expression("[1, 2, 3", values=SymbolTable()) expected = [ - "Syntax error: '[' was never closed\n", + "Syntax error: unexpected EOF while parsing\n", " [1, 2, 3\n", " ^~~~~~~~", ] @@ -332,15 +338,48 @@ def test_unclosed_string(self): with pytest.raises(ExpressionError) as exc_info: evaluate_expression("'unclosed", values=SymbolTable()) expected = [ - "Syntax error: unterminated string literal (detected at line 1)\n", + "Syntax error: missing closing quote in string literal\n", " 'unclosed\n", " ^~~~~~~~~", ] assert str(exc_info.value) == "".join(expected) + @pytest.mark.parametrize( + "expr, expected", + [ + pytest.param( + "1 +", "Syntax error: Expected an expression\n 1 +\n ^~~", id="no operand" + ), + pytest.param( + "f(a b)", + "Syntax error: Expected `,`, found identifier\n f(a b)\n ^", + id="missing comma", + ), + pytest.param( + "Param.A\nParam.B", + "Syntax error: Unexpected token at the end of an expression\n Param.B\n ^~~~~", + id="two lines", + ), + ], + ) + def test_ruff_wording(self, expr: str, expected: str) -> None: + """More of Ruff's wording (openjd-rs#531). Before, these were + "invalid syntax", "invalid syntax. Perhaps you forgot a comma?" with a + caret on ``a b``, and, for two lines, the same comma hint on the first + line, because the parser saw them wrapped in parentheses.""" + with pytest.raises(ExpressionError) as exc_info: + evaluate_expression(expr, values=SymbolTable()) + assert str(exc_info.value) == expected + class TestMultiLineExpressions: - """Tests for error formatting in multi-line expressions.""" + """Tests for error formatting in multi-line expressions. + + The caret sits on the operator, as it does for a single-line expression. + Since openjd-expr 0.15.0 (openjd-rs#531) a multi-line expression is parsed + as written rather than wrapped in parentheses. Before, the wrap shifted + the caret one character right of the operator (``~~~^~~~``). + """ def test_error_in_parentheses_multiline(self): """Error on second line of parenthesized expression.""" @@ -349,7 +388,7 @@ def test_error_in_parentheses_multiline(self): expected = [ "Cannot use '+' operator with int and string\n", " 1 + 'x'\n", - " ~~~^~~~", + " ~~^~~~~", ] assert str(exc_info.value) == "".join(expected) @@ -360,7 +399,7 @@ def test_error_in_list_multiline(self): expected = [ "Cannot use '+' operator with int and string\n", " 2 + 'x',\n", - " ~~~^~~~", + " ~~^~~~~", ] assert str(exc_info.value) == "".join(expected) @@ -371,7 +410,7 @@ def test_error_on_first_line_multiline(self): expected = [ "Cannot use '+' operator with int and string\n", " 1 + 'x' + (\n", - " ~~~^~~~", + " ~~^~~~~", ] assert str(exc_info.value) == "".join(expected) @@ -382,7 +421,7 @@ def test_deeply_nested_multiline(self): expected = [ "Cannot use '+' operator with int and string\n", " 1 + 'x'\n", - " ~~~^~~~", + " ~~^~~~~", ] assert str(exc_info.value) == "".join(expected) diff --git a/test/openjd/expr/test_parser_work_bound.py b/test/openjd/expr/test_parser_work_bound.py new file mode 100644 index 00000000..4e5dc1d9 --- /dev/null +++ b/test/openjd/expr/test_parser_work_bound.py @@ -0,0 +1,129 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Parsing an invalid expression takes bounded time (openjd-rs#489). + +openjd-expr 0.11.0 to 0.14.0 parsed with RustPython's fork of Ruff's parser +(openjd-rs#429), whose CPython-style syntax-error hints parse ahead and rewind. +Hints nest inside brackets, so parse time doubled per unclosed bracket level. +Through these bindings, ``decode_job_template`` took 2.6 s in a release build +(39 s in a debug build) on the 101-byte ``"[i f(" * 20 + "x"``, and 5.0 s on +the 363-byte fuzz reproducer below. Any template field that takes an +expression could stall template decoding. openjd-expr 0.15.0 (openjd-rs#531) +parses with Astral's ``ruff_python_parser``, which has no such hints; each +input here now fails in a few milliseconds. + +Each case runs through the expression API and through both template decoders, +since the pure-Python ``openjd.model`` evaluates EXPR expressions with the same +Rust parser. The 1 s bound is generous so the tests don't flake on a slow CI +runner; one decode runs first so that one-time setup isn't timed. +""" + +from __future__ import annotations + +import time +from typing import Any, Callable + +import pytest + +from openjd.expr import ExpressionError, FormatString, parse_expression +from openjd.model import DecodeValidationError as V0DecodeValidationError +from openjd.model import decode_job_template as v0_decode_job_template +from openjd.model._v1 import decode_job_template as v1_decode_job_template +from openjd.model._v1.errors import DecodeValidationError as V1DecodeValidationError + +_TIME_BOUND_SECONDS = 1.0 + +# The fuzz harness's reproducer for openjd-rs#489, 363 bytes. +_SLOW_DOTS = ( + "ginaen([manifestVersion.......................(............(......................" + "........join.................(.....................n.................(..........." + "................destination_55555path.(..............data......................" + "......@....so, 1],.so[vted().d(......destination_55555path.(..............data.." + "..............................so, dx in e" +) + +_CASES = [ + pytest.param("[i f(" * 20 + "x", "Expected `,`, found identifier", id="unclosed [i f( x20"), + pytest.param("f(a " * 20 + "x", "Expected `,`, found identifier", id="unclosed f(a x20"), + pytest.param(_SLOW_DOTS, "Expected `,`, found `...`", id="fuzz slow-dots"), +] + + +def _template(expr: str) -> dict[str, Any]: + return { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "steps": [ + { + "name": "S", + "script": { + "actions": {"onRun": {"command": "echo", "args": ["{{ " + expr + " }}"]}} + }, + } + ], + } + + +def _timed(f: Callable[[], object], exc_type: type[Exception]) -> tuple[float, str]: + start = time.perf_counter() + with pytest.raises(exc_type) as exc_info: + f() + return time.perf_counter() - start, str(exc_info.value) + + +@pytest.fixture(scope="module", autouse=True) +def _warm_up() -> None: + v0_decode_job_template(template=_template("1"), supported_extensions=["EXPR"]) + v1_decode_job_template(template=_template("1"), supported_extensions=["EXPR"]) + + +def test_the_101_byte_reproducer_is_101_bytes() -> None: + """The input size the module docstring quotes.""" + assert len("[i f(" * 20 + "x") == 101 + assert len(_SLOW_DOTS) == 363 + + +@pytest.mark.parametrize("expr, headline", _CASES) +def test_parse_expression(expr: str, headline: str) -> None: + elapsed, message = _timed(lambda: parse_expression(expr), ExpressionError) + assert message.startswith(f"Syntax error: {headline}\n {expr}\n") + assert elapsed < _TIME_BOUND_SECONDS + + +@pytest.mark.parametrize("expr, headline", _CASES) +def test_format_string(expr: str, headline: str) -> None: + elapsed, message = _timed(lambda: FormatString("{{ " + expr + " }}"), ExpressionError) + assert f"Syntax error: {headline}\n" in message + assert elapsed < _TIME_BOUND_SECONDS + + +@pytest.mark.parametrize("expr, headline", _CASES) +def test_v0_decode_job_template(expr: str, headline: str) -> None: + """The pure-Python model, which the Python CLI uses.""" + elapsed, message = _timed( + lambda: v0_decode_job_template(template=_template(expr), supported_extensions=["EXPR"]), + V0DecodeValidationError, + ) + assert message.startswith( + "1 validation errors for JobTemplate\n" + "steps[0] -> script -> actions -> onRun -> args[0]:\n" + f"\tFailed to parse interpolation expression at [0, {len(expr) + 6}]. " + f"Reason: Syntax error: {headline}\n" + ) + assert elapsed < _TIME_BOUND_SECONDS + + +@pytest.mark.parametrize("expr, headline", _CASES) +def test_v1_decode_job_template(expr: str, headline: str) -> None: + elapsed, message = _timed( + lambda: v1_decode_job_template(template=_template(expr), supported_extensions=["EXPR"]), + V1DecodeValidationError, + ) + assert message.startswith( + "'jobtemplate-2023-09' failed checks: " + f"Failed to parse interpolation expression at [0, {len(expr) + 6}]. " + f"Reason: Syntax error: {headline}\n" + ) + assert elapsed < _TIME_BOUND_SECONDS diff --git a/test/openjd/expr/test_parsing.py b/test/openjd/expr/test_parsing.py index 05021420..5d394761 100644 --- a/test/openjd/expr/test_parsing.py +++ b/test/openjd/expr/test_parsing.py @@ -21,6 +21,29 @@ def test_keyword_in_expression_context(self) -> None: symtab = SymbolTable({"Param.if": 10, "Param.else": 20}) assert evaluate_expression("Param.if + Param.else", values=symtab).item() == 30 + def test_placeholder_never_equals_an_nfkc_normalized_name(self) -> None: + """A keyword attribute is renamed to a placeholder before parsing, and + ``Param.is`` gets ``bs``. The parser NFKC-normalizes identifiers, so + fullwidth ``Param.bs`` is the name ``bs`` too. Since openjd-expr 0.15.0 + (openjd-rs#534) the placeholder is checked against normalized names; + before, this gave ``"isis"``.""" + symtab = SymbolTable({"Param.bs": "fullwidth-bs", "Param.is": "is"}) + assert evaluate_expression("Param.bs + Param.is", values=symtab).item() == ( + "fullwidth-bsis" + ) + + def test_placeholder_when_every_first_letter_is_taken(self) -> None: + """The string literal holds every first-letter candidate for ``if`` + (``af`` to ``zf``). Before openjd-expr 0.15.0 (openjd-rs#534, + openjd-rs#333) that made the placeholder fall back to ``xx`` without a + check, so ``Param.xx`` read ``Param.if``'s value and this gave + ``[1, 1, 52]``. Now text in a string literal doesn't block a candidate, + and a candidate is checked against every name the parser sees.""" + symtab = SymbolTable({"Param.if": 1, "Param.xx": 2}) + names = "".join(c + "f" for c in "abcdefghijklmnopqrstuvwxyz") + result = evaluate_expression(f"[Param.if, Param.xx, len('{names}')]", values=symtab) + assert result.item() == [1, 2, 52] + def test_keyword_with_conditional(self) -> None: symtab = SymbolTable({"Param.if": 100}) assert evaluate_expression("Param.if if True else 0", values=symtab).item() == 100 diff --git a/test/openjd/model_v1/test_create_job.py b/test/openjd/model_v1/test_create_job.py index 5607c8bb..c55627f0 100644 --- a/test/openjd/model_v1/test_create_job.py +++ b/test/openjd/model_v1/test_create_job.py @@ -3017,3 +3017,105 @@ def test_a_value_that_is_not_an_integer_fails(self, value: str) -> None: assert ( str(excinfo.value) == f"Task parameter 'F' range[1]: value '{value}' is not an integer" ) + + +def _preprocess(parameter_definitions: list[dict], values: dict, tmp_path: Path) -> dict: + """Decode a one-step template with ``parameter_definitions`` (with EXPR, for + the list types) and return ``preprocess_job_parameters``' values as text.""" + job_template = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "parameterDefinitions": parameter_definitions, + "steps": minimal_steps_v2023_09, + }, + supported_extensions=["EXPR"], + ) + result = preprocess_job_parameters( + job_template=job_template, + job_parameter_values=values, + job_template_dir=tmp_path, + current_working_dir=tmp_path, + ) + return {name: value.value for name, value in result.items()} + + +class TestFloatGivenAsANumber: + """Since openjd-model 0.16.0 (openjd-rs#539) a FLOAT given as a YAML or JSON + number is written the way Python's ``repr(float)`` writes it, with zero + always ``0.0`` (Expression Language §1.3.4). A FLOAT given as a string keeps + its spelling (Template Schemas §7.5). Before, the number's own digits were + written, so ``2`` stayed ``2`` and ``1e20`` became ``100000000000000000000``. + """ + + @pytest.mark.parametrize( + "default, expected", + [ + pytest.param(2, "2.0", id="int"), + pytest.param(1e20, "1e+20", id="large"), + pytest.param(0.00001, "1e-05", id="small"), + pytest.param(0, "0.0", id="zero"), + pytest.param(2.5, "2.5", id="plain"), + pytest.param("2", "2", id="string keeps its spelling"), + ], + ) + def test_default(self, default: Any, expected: str, tmp_path: Path) -> None: + defs = [{"name": "F", "type": "FLOAT", "default": default}] + assert _preprocess(defs, {}, tmp_path) == {"F": expected} + + @pytest.mark.parametrize("value", [2, 2.0]) + def test_value(self, value: Any, tmp_path: Path) -> None: + assert _preprocess([{"name": "F", "type": "FLOAT"}], {"F": value}, tmp_path) == {"F": "2.0"} + + def test_list_default_items(self, tmp_path: Path) -> None: + defs = [{"name": "L", "type": "LIST[FLOAT]", "default": [2, 1e20]}] + assert _preprocess(defs, {}, tmp_path) == {"L": "[2.0, 1e+20]"} + + +class TestNumericStringsInListParameters: + """openjd-model 0.16.0 (openjd-rs#538). A list value given as a native + Python list of numeric strings is coerced item by item, as the same value + given as JSON text is (Template Schemas §2.13, §2.14, §2.16). Before, a + LIST[INT] or LIST[FLOAT] list raised ``Cannot coerce list to LIST[INT]``, + and a LIST[LIST[INT]] one kept its items as strings. A LIST[FLOAT] default + item written as a string keeps its spelling; before, ``"2.50"`` became + ``2.5``.""" + + @pytest.mark.parametrize( + "param_type, value, expected", + [ + pytest.param("LIST[INT]", ["1", " 2"], "[1, 2]", id="LIST[INT]"), + pytest.param("LIST[FLOAT]", ["1.50", " 2"], "[1.50, 2]", id="LIST[FLOAT]"), + pytest.param("LIST[LIST[INT]]", [["1", " 2"]], "[[1, 2]]", id="LIST[LIST[INT]]"), + ], + ) + def test_native_list_value( + self, param_type: str, value: list, expected: str, tmp_path: Path + ) -> None: + defs = [{"name": "L", "type": param_type}] + assert _preprocess(defs, {"L": value}, tmp_path) == {"L": expected} + + def test_a_bad_item_names_its_position(self, tmp_path: Path) -> None: + with pytest.raises(DecodeValidationError) as exc_info: + _preprocess([{"name": "L", "type": "LIST[INT]"}], {"L": ["1", "x"]}, tmp_path) + assert str(exc_info.value) == ( + "Parameter 'L': item[1]: Value 'x' is not a valid integer or integer string." + ) + + def test_list_float_default_string_item_keeps_its_spelling(self, tmp_path: Path) -> None: + defs = [{"name": "L", "type": "LIST[FLOAT]", "default": [1.50, "2.50"]}] + assert _preprocess(defs, {}, tmp_path) == {"L": "[1.5, 2.50]"} + + +class TestWhiteSpaceAroundNumericStrings: + """Since openjd-model 0.16.0 (openjd-rs#530) white space around a number in + a string value for an INT or FLOAT parameter is ignored (Template Schemas + §7.5). Before, ``" 7 "`` was ``not a valid integer``.""" + + def test_int(self, tmp_path: Path) -> None: + assert _preprocess([{"name": "I", "type": "INT"}], {"I": " 7 "}, tmp_path) == {"I": "7"} + + def test_float_keeps_its_spelling(self, tmp_path: Path) -> None: + defs = [{"name": "F", "type": "FLOAT"}] + assert _preprocess(defs, {"F": " 1.50 "}, tmp_path) == {"F": "1.50"} diff --git a/test/openjd/model_v1/test_step_param_space_iter.py b/test/openjd/model_v1/test_step_param_space_iter.py index 2a7d2eca..b611c688 100644 --- a/test/openjd/model_v1/test_step_param_space_iter.py +++ b/test/openjd/model_v1/test_step_param_space_iter.py @@ -658,6 +658,51 @@ def test_an_unparseable_element_is_still_rejected(self) -> None: self._rendered(["not-a-float"]) +class TestWholeFieldFloatRangeExpressionKeepsSpelling: + """A FLOAT task parameter whose ``range`` is one whole-field expression gives + each float the spelling it carries, as a range list element does (Expression + Language §1.3.4). openjd-model 0.16.0 (openjd-rs#540); before, the list was + rebuilt from the numbers, so ``["4.20", "5.30"]`` rendered ``4.2`` and + ``5.3``. (Before openjd-model 0.16.0, openjd-rs#538, a native list of + numeric strings couldn't be given for LIST[FLOAT] at all.)""" + + @staticmethod + def _rendered(parameter_definitions: list[dict], rng: Any, values: dict) -> list[str]: + template = { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "parameterDefinitions": parameter_definitions, + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [{"name": "F", "type": "FLOAT", "range": rng}] + }, + "script": { + "actions": {"onRun": {"command": "echo", "args": ["{{Task.Param.F}}"]}} + }, + } + ], + } + job = create_job( + job_template=decode_job_template(template=template, supported_extensions=["EXPR"]), + job_parameter_values=values, + ) + return [params["F"].value for params in StepParameterSpaceIterator(step=job.steps[0])] + + @pytest.mark.parametrize( + "rng", ["{{ Param.LF }}", ["{{ Param.LF }}"]], ids=["whole field", "list element"] + ) + def test_a_list_parameter(self, rng: Any) -> None: + defs = [{"name": "LF", "type": "LIST[FLOAT]"}] + assert self._rendered(defs, rng, {"LF": ["4.20", "5.30"]}) == ["4.20", "5.30"] + + def test_a_float_parameter_in_a_list_expression(self) -> None: + defs = [{"name": "X", "type": "FLOAT"}] + assert self._rendered(defs, "{{ [Param.X] }}", {"X": "1.50"}) == ["1.50"] + + class TestChunkIntContains: """``__contains__`` must round-trip values yielded by a chunked iterator. The iterator yields ``TaskParameterValue`` instances diff --git a/test/openjd/sessions/test_failed_actions.py b/test/openjd/sessions/test_failed_actions.py new file mode 100644 index 00000000..3c7d00e4 --- /dev/null +++ b/test/openjd/sessions/test_failed_actions.py @@ -0,0 +1,153 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""How ``Session`` handles a failed action, through openjd-sessions 0.10.0. + +- openjd-rs#532: when ``run_task`` or ``enter_environment`` fails before its + subprocess starts, for example because a parameter value can't be coerced, + the action fails with the error's message and the session becomes + ending-only, as the pure-Python ``openjd.sessions`` does. Before, the binding + saw an error it discards, so ``action_status`` stayed ``None`` and the + session went back to ``READY``. +- openjd-rs#524: an action that fails leaves the session ending-only, even + after ``exit_environment(keep_session_running=True)`` succeeds. Before, that + exit returned the session to ``READY``. +- openjd-rs#536: when ``exit_environment`` can't build its symbol table from + the ``resolved_symtab`` it is given, it runs ``onExit`` with the table the + environment was entered with. Before, ``onExit`` didn't run. + +What a session does with a call it then refuses is in ``test_known_gaps.py``. +""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path +from typing import Any + +from openjd._openjd_rs import ( + ActionState, + Session, + SessionState, + _reconstruct_serialized_symtab, + create_environment, + decode_environment_template, + deserialize_step, +) + + +def _python(code: str) -> dict[str, Any]: + return {"command": sys.executable, "args": ["-c", code]} + + +def _step_script(action: dict[str, Any]) -> Any: + return deserialize_step({"name": "S", "script": {"actions": {"onRun": action}}}).script + + +def _environment(**actions: dict[str, Any]) -> Any: + return create_environment( + decode_environment_template( + template={ + "specificationVersion": "environment-2023-09", + "environment": {"name": "E", "script": {"actions": actions}}, + } + ) + ) + + +def _settle(session: Session) -> None: + deadline = time.monotonic() + 30 + while session.state == SessionState.RUNNING and time.monotonic() < deadline: + time.sleep(0.05) + assert session.state != SessionState.RUNNING, "action did not finish within 30s" + + +def _session(tmp_path: Path, **kwargs: Any) -> Session: + return Session( + session_id="s", job_parameter_values={}, session_root_directory=str(tmp_path), **kwargs + ) + + +def test_a_task_that_fails_before_it_starts_fails_the_action(tmp_path: Path) -> None: + session = _session(tmp_path) + try: + session.run_task( + step_script=_step_script({"command": "echo", "args": ["{{Task.Param.F}}"]}), + task_parameter_values={"F": {"type": "INT", "value": "abc"}}, + ) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message == ( + "Failed to coerce parameter 'Task.RawParam.F': " + "Cannot convert 'abc' to int: invalid digit found in string" + ) + assert session.state == SessionState.READY_ENDING + finally: + session.cleanup() + + +def test_an_environment_that_fails_before_it_starts_fails_the_action(tmp_path: Path) -> None: + """The environment still counts as entered, so it can be exited.""" + session = Session( + session_id="s", + job_parameter_values={"P": {"type": "PATH", "value": "/a/" + "x" * 32_800}}, + session_root_directory=str(tmp_path), + ) + try: + identifier = session.enter_environment(environment=_environment(onEnter=_python(""))) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message == ( + "Failed to apply path mapping to Param.P: " + "the path is 32803 bytes long, more than the limit of 32768 bytes" + ) + assert session.state == SessionState.READY_ENDING + assert session.environments_entered == [identifier] + finally: + session.cleanup() + + +def test_a_failed_task_keeps_the_session_ending_only(tmp_path: Path) -> None: + session = _session(tmp_path) + try: + identifier = session.enter_environment(environment=_environment(onEnter=_python(""))) + _settle(session) + session.run_task(step_script=_step_script(_python("raise SystemExit(1)"))) + _settle(session) + assert session.state == SessionState.READY_ENDING + session.exit_environment(identifier=identifier, keep_session_running=True) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert session.state == SessionState.READY_ENDING + finally: + session.cleanup() + + +def test_on_exit_runs_with_the_enter_time_table(tmp_path: Path) -> None: + marker = tmp_path / "on-exit-ran" + session = _session(tmp_path) + try: + identifier = session.enter_environment( + environment=_environment( + onEnter=_python(""), + onExit=_python(f"open({str(marker)!r}, 'w').close()"), + ) + ) + _settle(session) + not_a_table = _reconstruct_serialized_symtab("{}") + session.exit_environment(identifier=identifier, resolved_symtab=not_a_table) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert marker.exists() + assert session.state == SessionState.READY + finally: + session.cleanup() diff --git a/test/openjd/sessions/test_known_gaps.py b/test/openjd/sessions/test_known_gaps.py new file mode 100644 index 00000000..39fdcdb3 --- /dev/null +++ b/test/openjd/sessions/test_known_gaps.py @@ -0,0 +1,60 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Known gaps between the sessions binding and the pure-Python reference. + +Each test is a strict xfail. When a gap is fixed, move its test next to the +surface it covers rather than leaving it here. +""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path + +import pytest + +from openjd._openjd_rs import Session, SessionState, deserialize_step + +# ── A call the session refuses returns as if it had started an action ── +# +# The non-blocking actions in ``rust-bindings/src/sessions/session.rs`` +# (``run_task``, ``enter_environment``, ``exit_environment``, +# ``run_subprocess``) hand the call to a background thread and discard the +# crate's ``Result``. A call the crate refuses up front, such as ``run_task`` +# on an ending-only session, therefore returns ``None``, runs nothing, and +# leaves ``action_status`` showing the previous action. The pure-Python +# ``openjd.sessions.Session`` raises for a session that isn't ``READY``. +# +# openjd-sessions 0.10.0 (openjd-rs#524) made this reachable in normal use: a +# failed action keeps the session ending-only even after +# ``exit_environment(keep_session_running=True)``, so the next ``run_task`` +# is refused, and the caller sees the exit's ``SUCCESS``. With openjd-sessions +# 0.9.0 that ``run_task`` ran. + + +@pytest.mark.xfail( + reason="A run_task the crate refuses returns None and leaves the previous action_status", + strict=True, +) +def test_run_task_on_an_ending_only_session_raises(tmp_path: Path) -> None: + def script(code: str): + return deserialize_step( + { + "name": "S", + "script": {"actions": {"onRun": {"command": sys.executable, "args": ["-c", code]}}}, + } + ).script + + session = Session(session_id="s", job_parameter_values={}, session_root_directory=str(tmp_path)) + try: + session.run_task(step_script=script("raise SystemExit(1)")) + deadline = time.monotonic() + 30 + while session.state == SessionState.RUNNING and time.monotonic() < deadline: + time.sleep(0.05) + assert session.state == SessionState.READY_ENDING + with pytest.raises(Exception): + session.run_task(step_script=script("")) + finally: + session.cleanup() From 991fec3ae5d1447fb368a69778748939271a55f4 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:09:31 +0000 Subject: [PATCH 10/21] chore: Regenerate THIRD-PARTY-LICENSES.txt for the openjd-rs 0.15/0.16/0.10 bump Regenerated with scripts/check_third_party_licenses.sh --update. The openjd crates' versions change, Astral's ruff_python_* 0.0.21 and ruff_unicode_names2 replace rustpython-ruff_python_* 0.16.10 and unicode_names2, phf moves to 0.14, and the icu_* crates and their Unicode-3.0 license text, synstructure, displaydoc, stable_deref_trait and utf8_iter drop out. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- THIRD-PARTY-LICENSES.txt | 103 +++++---------------------------------- 1 file changed, 12 insertions(+), 91 deletions(-) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index f1a83782..8040920f 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -611,7 +611,6 @@ PERFORMANCE OF THIS SOFTWARE. ** nohash-hasher; version 0.2.0 -- https://crates.io/crates/nohash-hasher ** static_assertions; version 1.1.0 -- https://crates.io/crates/static_assertions ** tinyvec; version 1.12.0 -- https://crates.io/crates/tinyvec -** utf8_iter; version 1.0.4 -- https://crates.io/crates/utf8_iter Apache License Version 2.0, January 2004 @@ -2094,7 +2093,6 @@ limitations under the License. ** bitflags; version 2.13.1 -- https://crates.io/crates/bitflags ** bstr; version 1.13.1 -- https://crates.io/crates/bstr ** cfg-if; version 1.0.4 -- https://crates.io/crates/cfg-if -** displaydoc; version 0.2.7 -- https://crates.io/crates/displaydoc ** drop_bomb; version 0.1.5 -- https://crates.io/crates/drop_bomb ** either; version 1.18.0 -- https://crates.io/crates/either ** encoding_rs_io; version 0.1.8 -- https://crates.io/crates/encoding_rs_io @@ -2112,13 +2110,12 @@ limitations under the License. ** regex-automata; version 0.4.18 -- https://crates.io/crates/regex-automata ** regex-syntax; version 0.8.11 -- https://crates.io/crates/regex-syntax ** regex; version 1.13.1 -- https://crates.io/crates/regex +** ruff_unicode_names2; version 0.0.21 -- https://crates.io/crates/ruff_unicode_names2 ** signal-hook-registry; version 1.4.8 -- https://crates.io/crates/signal-hook-registry ** smallvec; version 1.15.2 -- https://crates.io/crates/smallvec -** stable_deref_trait; version 1.2.1 -- https://crates.io/crates/stable_deref_trait ** stacker; version 0.1.25 -- https://crates.io/crates/stacker ** unicode-normalization; version 0.1.25 -- https://crates.io/crates/unicode-normalization ** unicode-width; version 0.2.2 -- https://crates.io/crates/unicode-width -** unicode_names2; version 1.3.0 -- https://crates.io/crates/unicode_names2 ** uuid; version 1.26.0 -- https://crates.io/crates/uuid ** value-bag; version 1.13.2 -- https://crates.io/crates/value-bag ** wasi; version 0.11.1+wasi-snapshot-preview1 -- https://crates.io/crates/wasi @@ -2743,9 +2740,9 @@ limitations under the License. ** itoa; version 1.0.18 -- https://crates.io/crates/itoa ** libc; version 0.2.189 -- https://crates.io/crates/libc ** manyhow-macros; version 0.14.1 -- https://crates.io/crates/manyhow-macros -** openjd-expr; version 0.14.0 -- https://crates.io/crates/openjd-expr -** openjd-model; version 0.15.0 -- https://crates.io/crates/openjd-model -** openjd-sessions; version 0.9.0 -- https://crates.io/crates/openjd-sessions +** openjd-expr; version 0.15.0 -- https://crates.io/crates/openjd-expr +** openjd-model; version 0.16.0 -- https://crates.io/crates/openjd-model +** openjd-sessions; version 0.10.0 -- https://crates.io/crates/openjd-sessions ** pin-project-lite; version 0.2.17 -- https://crates.io/crates/pin-project-lite ** portable-atomic; version 1.15.0 -- https://crates.io/crates/portable-atomic ** proc-macro2; version 1.0.107 -- https://crates.io/crates/proc-macro2 @@ -3074,17 +3071,6 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------- -** synstructure; version 0.14.0 -- https://crates.io/crates/synstructure -Copyright 2016 Nika Layzell - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - ------ ** tokio-macros; version 2.7.2 -- https://crates.io/crates/tokio-macros MIT License @@ -3212,11 +3198,11 @@ SOFTWARE. ------ -** rustpython-ruff_python_ast; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_python_ast -** rustpython-ruff_python_parser; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_python_parser -** rustpython-ruff_python_trivia; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_python_trivia -** rustpython-ruff_source_file; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_source_file -** rustpython-ruff_text_size; version 0.16.10 -- https://crates.io/crates/rustpython-ruff_text_size +** ruff_python_ast; version 0.0.21 -- https://crates.io/crates/ruff_python_ast +** ruff_python_parser; version 0.0.21 -- https://crates.io/crates/ruff_python_parser +** ruff_python_trivia; version 0.0.21 -- https://crates.io/crates/ruff_python_trivia +** ruff_source_file; version 0.0.21 -- https://crates.io/crates/ruff_source_file +** ruff_text_size; version 0.0.21 -- https://crates.io/crates/ruff_text_size MIT License Copyright (c) @@ -3291,8 +3277,8 @@ DEALINGS IN THE SOFTWARE. ------ -** phf; version 0.11.3 -- https://crates.io/crates/phf -** phf_shared; version 0.11.3 -- https://crates.io/crates/phf_shared +** phf; version 0.14.0 -- https://crates.io/crates/phf +** phf_shared; version 0.14.0 -- https://crates.io/crates/phf_shared The MIT License (MIT) Copyright (c) 2014-2022 Steven Fackler, Yuki Okushi @@ -3410,72 +3396,7 @@ authorization of the copyright holder. ------ -** icu_collections; version 2.3.0 -- https://crates.io/crates/icu_collections -** icu_locale_core; version 2.3.0 -- https://crates.io/crates/icu_locale_core -** icu_properties; version 2.3.0 -- https://crates.io/crates/icu_properties -** icu_properties_data; version 2.3.0 -- https://crates.io/crates/icu_properties_data -** icu_provider; version 2.3.1 -- https://crates.io/crates/icu_provider -** litemap; version 0.8.3 -- https://crates.io/crates/litemap -** potential_utf; version 0.1.6 -- https://crates.io/crates/potential_utf -** tinystr; version 0.8.4 -- https://crates.io/crates/tinystr -** writeable; version 0.6.4 -- https://crates.io/crates/writeable -** yoke-derive; version 0.8.4 -- https://crates.io/crates/yoke-derive -** yoke; version 0.8.3 -- https://crates.io/crates/yoke -** zerofrom-derive; version 0.1.8 -- https://crates.io/crates/zerofrom-derive -** zerofrom; version 0.1.8 -- https://crates.io/crates/zerofrom -** zerotrie; version 0.2.5 -- https://crates.io/crates/zerotrie -** zerovec-derive; version 0.11.6 -- https://crates.io/crates/zerovec-derive -** zerovec; version 0.11.8 -- https://crates.io/crates/zerovec -UNICODE LICENSE V3 - -COPYRIGHT AND PERMISSION NOTICE - -Copyright © 2020-2024 Unicode, Inc. - -NOTICE TO USER: Carefully read the following legal agreement. BY -DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR -SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE -TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT -DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. - -Permission is hereby granted, free of charge, to any person obtaining a -copy of data files and any associated documentation (the "Data Files") or -software and any associated documentation (the "Software") to deal in the -Data Files or Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, and/or sell -copies of the Data Files or Software, and to permit persons to whom the -Data Files or Software are furnished to do so, provided that either (a) -this copyright and permission notice appear with all copies of the Data -Files or Software, or (b) this copyright and permission notice appear in -associated Documentation. - -THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF -THIRD PARTY RIGHTS. - -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE -BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, -OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, -ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA -FILES OR SOFTWARE. - -Except as contained in this notice, the name of a copyright holder shall -not be used in advertising or otherwise to promote the sale, use or other -dealings in these Data Files or Software without prior written -authorization of the copyright holder. - -SPDX-License-Identifier: Unicode-3.0 - -— - -Portions of ICU4X may have been adapted from ICU4C and/or ICU4J. -ICU 1.8.1 to ICU 57.1 © 1995-2016 International Business Machines Corporation and others. - - ------- -** unicode_names2; version 1.3.0 -- https://crates.io/crates/unicode_names2 +** ruff_unicode_names2; version 0.0.21 -- https://crates.io/crates/ruff_unicode_names2 UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE Unicode Data Files include all data files under the directories http://www.unicode.org/Public/, http://www.unicode.org/reports/, http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and http://www.unicode.org/utility/trac/browser/. From 31f47b0a33a54651ff7dd916f04cd84e44fb992c Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:22:19 +0000 Subject: [PATCH 11/21] test: Correct the bump's claims and pin white space in session parameters A review of the openjd-rs 0.15/0.16/0.10 bump found: - Through a Session, white space around a number in an INT or FLOAT job or task parameter value is now ignored (openjd-rs#530). Before, the value failed before the action started and the binding didn't report it, so the task silently didn't run. Pin it, and say so in the sessions spec. - Only a FLOAT parameter's default given as a number changed its text with openjd-rs#539 (2 was written "2", now "2.0"). A value given as a Python number and LIST[FLOAT] default items were already written as repr writes them. The previous commit's message, the FLOAT test class and the model spec read as if all three changed; narrow them. - The run_task known gap predates openjd-sessions 0.10.0: a run_task straight after a failed one was refused the same way with 0.9.0. openjd-rs#524 adds a path into it, after a successful exit_environment(keep_session_running=True). Reword the comment, and record the gap in the sessions quality report. - Cargo.lock adds phf 0.14 beside phf 0.11, which the stub-gen feature still pulls in; the previous commit said phf moved to 0.14. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- ...ions-bindings-quality-evaluation-report.md | 1 + specs/python-model-interface.md | 5 +-- specs/python-sessions-interface.md | 4 ++- test/openjd/model_v1/test_create_job.py | 14 +++++--- test/openjd/sessions/test_failed_actions.py | 35 +++++++++++++++++++ test/openjd/sessions/test_known_gaps.py | 12 ++++--- 6 files changed, 58 insertions(+), 13 deletions(-) diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index 08b3fe47..98f30444 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -370,6 +370,7 @@ counted as test-suite failures. | Failure | Reference | Binding | Status | |---|---|---|---| | `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `RuntimeError("An action is already running")` | ⚠ message change | +| `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | returns `None`, runs nothing, `action_status` still shows the previous action (the binding discards the crate's `Err`); openjd-sessions 0.10.0 keeps a session `READY_ENDING` after a failed action even when an exit keeps it running | ❌ silent no-op (strict xfail in `test/openjd/sessions/test_known_gaps.py`) | | `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change | | `cancel_action` while running | succeeds | `RuntimeError("Cannot cancel: session is busy with an action")` | ❌ contract inversion | | `Session(session_root_directory=…)` non-existent dir | `RuntimeError("Ensure that the root directory ...")` | likely Rust I/O error → `SessionError(...)` (untested) | ⚠ untested | diff --git a/specs/python-model-interface.md b/specs/python-model-interface.md index dc7b83df..017d4539 100644 --- a/specs/python-model-interface.md +++ b/specs/python-model-interface.md @@ -290,8 +290,9 @@ constraints are checked against the joined value. URI values are refused. Violat A FLOAT given as a number, as a default, a `LIST[FLOAT]` item or a value, is written the way Python's `repr(float)` writes it, with zero always `0.0`: `default: 2` gives `"2.0"` and `1e20` gives `"1e+20"`. A -FLOAT given as a string keeps its spelling (Template Schemas §7.5, -openjd-rs#539). White space around a number in a string value is +FLOAT given as a string keeps its spelling (Template Schemas §7.5). +Before openjd-model 0.16.0 (openjd-rs#539), a scalar FLOAT `default` +given as a number kept its own digits (`2`, `100000000000000000000`). White space around a number in a string value is ignored, so `" 7 "` is the INT `7` (openjd-rs#530). A `LIST[INT]`, `LIST[FLOAT]` or `LIST[LIST[INT]]` value given as a Python list of numeric strings is coerced item by item, the same as the JSON text of diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index 9c4654d6..c44102e8 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -206,7 +206,9 @@ coerced, the action is `FAILED` with the error's message, as in the pure-Python reference (openjd-rs#532); an environment that fails this way still counts as entered. When `exit_environment` can't build its symbol table from `resolved_symtab`, it runs `onExit` with the table the -environment was entered with (openjd-rs#536). +environment was entered with (openjd-rs#536). White space around a +number in an INT or FLOAT job or task parameter value is ignored, so +`" 7 "` is `7` and `" 1.50 "` is `1.50` (openjd-rs#530). Known gap: a call the session refuses, such as `run_task` on a `READY_ENDING` session, returns `None` without running anything, and diff --git a/test/openjd/model_v1/test_create_job.py b/test/openjd/model_v1/test_create_job.py index c55627f0..56962a26 100644 --- a/test/openjd/model_v1/test_create_job.py +++ b/test/openjd/model_v1/test_create_job.py @@ -3042,11 +3042,15 @@ def _preprocess(parameter_definitions: list[dict], values: dict, tmp_path: Path) class TestFloatGivenAsANumber: - """Since openjd-model 0.16.0 (openjd-rs#539) a FLOAT given as a YAML or JSON - number is written the way Python's ``repr(float)`` writes it, with zero - always ``0.0`` (Expression Language §1.3.4). A FLOAT given as a string keeps - its spelling (Template Schemas §7.5). Before, the number's own digits were - written, so ``2`` stayed ``2`` and ``1e20`` became ``100000000000000000000``. + """A FLOAT given as a YAML or JSON number is written the way Python's + ``repr(float)`` writes it, with zero always ``0.0`` (Expression Language + §1.3.4). A FLOAT given as a string keeps its spelling (Template Schemas §7.5). + + What changed in openjd-model 0.16.0 (openjd-rs#539) is a FLOAT parameter's + ``default`` given as a number: before, its own digits were written, so ``2`` + stayed ``2`` and ``1e20`` became ``100000000000000000000``. A value given as + a Python number and a ``LIST[FLOAT]`` default's number items were already + written this way; ``test_value`` and ``test_list_default_items`` pin them. """ @pytest.mark.parametrize( diff --git a/test/openjd/sessions/test_failed_actions.py b/test/openjd/sessions/test_failed_actions.py index 3c7d00e4..141726f3 100644 --- a/test/openjd/sessions/test_failed_actions.py +++ b/test/openjd/sessions/test_failed_actions.py @@ -15,6 +15,9 @@ - openjd-rs#536: when ``exit_environment`` can't build its symbol table from the ``resolved_symtab`` it is given, it runs ``onExit`` with the table the environment was entered with. Before, ``onExit`` didn't run. +- openjd-rs#530: white space around a number in an INT or FLOAT parameter + value is ignored. Before, such a value failed before the action started, + which the binding didn't report, so the task silently didn't run. What a session does with a call it then refuses is in ``test_known_gaps.py``. """ @@ -151,3 +154,35 @@ def test_on_exit_runs_with_the_enter_time_table(tmp_path: Path) -> None: assert session.state == SessionState.READY finally: session.cleanup() + + +def test_white_space_around_a_numeric_parameter_value_is_ignored(tmp_path: Path) -> None: + out = tmp_path / "out.txt" + session = Session( + session_id="s", + job_parameter_values={"I": {"type": "INT", "value": " 7 "}}, + session_root_directory=str(tmp_path), + ) + try: + session.run_task( + step_script=_step_script( + { + "command": sys.executable, + "args": [ + "-c", + "import sys; open(sys.argv[1], 'w').write(sys.argv[2] + '|' + sys.argv[3])", + str(out), + "{{Param.I}}", + "{{Task.Param.F}}", + ], + } + ), + task_parameter_values={"F": {"type": "FLOAT", "value": " 1.50 "}}, + ) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert out.read_text() == "7|1.50" + finally: + session.cleanup() diff --git a/test/openjd/sessions/test_known_gaps.py b/test/openjd/sessions/test_known_gaps.py index 39fdcdb3..005717f2 100644 --- a/test/openjd/sessions/test_known_gaps.py +++ b/test/openjd/sessions/test_known_gaps.py @@ -27,11 +27,13 @@ # leaves ``action_status`` showing the previous action. The pure-Python # ``openjd.sessions.Session`` raises for a session that isn't ``READY``. # -# openjd-sessions 0.10.0 (openjd-rs#524) made this reachable in normal use: a -# failed action keeps the session ending-only even after -# ``exit_environment(keep_session_running=True)``, so the next ``run_task`` -# is refused, and the caller sees the exit's ``SUCCESS``. With openjd-sessions -# 0.9.0 that ``run_task`` ran. +# The gap predates openjd-sessions 0.10.0: a ``run_task`` straight after a +# failed one, as below, was refused the same way with 0.9.0. 0.10.0 +# (openjd-rs#524) adds a common path into it: a failed action now keeps the +# session ending-only even after a successful +# ``exit_environment(keep_session_running=True)``, so the next ``run_task`` is +# refused and the caller sees the exit's ``SUCCESS``. With 0.9.0 that exit +# returned the session to ``READY`` and the ``run_task`` ran. @pytest.mark.xfail( From 051b5b2cf748c6bcee827e7366f38bc8a9f07b75 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:28:03 +0000 Subject: [PATCH 12/21] test: Pin openjd-rs#539's float tie rounding and JSON float reading A second review of the bump found that openjd-rs#539 changed two more things, which the previous commit wrongly described as unchanged: - openjd-expr's float text now takes its digits from ryu, so a shortest-digit tie rounds to even as Python's repr does: 1000000000000000.25 is written 1000000000000000.2, where it was 1000000000000000.3. This reaches computed floats, FLOAT values given as a Python number, and LIST[FLOAT] items. - A LIST[FLOAT] value given as JSON text reads each number as the nearest double (serde_json float_roundtrip). Before, 2.2250738585072011e-308 read as 2.2250738585072014e-308. Pin both, correct the FLOAT test class docstring, and describe them in the expr and model specs. Also correct the sessions spec and quality report: on a READY_ENDING session, enter_environment returns an identifier that is never entered rather than None. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- ...ions-bindings-quality-evaluation-report.md | 2 +- specs/python-expr-interface.md | 5 +++ specs/python-model-interface.md | 12 ++++--- specs/python-sessions-interface.md | 8 +++-- test/openjd/expr/test_strings.py | 24 +++++++++++++ test/openjd/model_v1/test_create_job.py | 35 ++++++++++++++++--- 6 files changed, 73 insertions(+), 13 deletions(-) diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index 98f30444..3bc26be8 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -370,7 +370,7 @@ counted as test-suite failures. | Failure | Reference | Binding | Status | |---|---|---|---| | `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `RuntimeError("An action is already running")` | ⚠ message change | -| `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | returns `None`, runs nothing, `action_status` still shows the previous action (the binding discards the crate's `Err`); openjd-sessions 0.10.0 keeps a session `READY_ENDING` after a failed action even when an exit keeps it running | ❌ silent no-op (strict xfail in `test/openjd/sessions/test_known_gaps.py`) | +| `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | runs nothing (`run_task` returns `None`; `enter_environment` returns an identifier that was never entered), `action_status` still shows the previous action (the binding discards the crate's `Err`); openjd-sessions 0.10.0 keeps a session `READY_ENDING` after a failed action even when an exit keeps it running | ❌ silent no-op (strict xfail in `test/openjd/sessions/test_known_gaps.py`) | | `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change | | `cancel_action` while running | succeeds | `RuntimeError("Cannot cancel: session is busy with an action")` | ❌ contract inversion | | `Session(session_root_directory=…)` non-existent dir | `RuntimeError("Ensure that the root directory ...")` | likely Rust I/O error → `SessionError(...)` (untested) | ⚠ untested | diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index c7e9e3e8..a9a11b82 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -979,6 +979,11 @@ roughly linear in its length, whatever its bracket nesting. A multi-line expression is parsed as written, so a caret points into the text the author wrote. +A computed float is written as Python's `repr(float)` writes it, +including a shortest-digit tie, which rounds to even: +`string(1000000000000000.25 * 1.0)` is `"1000000000000000.2"` +(openjd-expr 0.15.0, openjd-rs#539; before, ties rounded up). + | Exception | Base | |---|---| diff --git a/specs/python-model-interface.md b/specs/python-model-interface.md index 017d4539..55b7b15f 100644 --- a/specs/python-model-interface.md +++ b/specs/python-model-interface.md @@ -289,10 +289,14 @@ constraints are checked against the joined value. URI values are refused. Violat A FLOAT given as a number, as a default, a `LIST[FLOAT]` item or a value, is written the way Python's `repr(float)` writes it, with zero -always `0.0`: `default: 2` gives `"2.0"` and `1e20` gives `"1e+20"`. A -FLOAT given as a string keeps its spelling (Template Schemas §7.5). -Before openjd-model 0.16.0 (openjd-rs#539), a scalar FLOAT `default` -given as a number kept its own digits (`2`, `100000000000000000000`). White space around a number in a string value is +always `0.0`: `default: 2` gives `"2.0"`, `1e20` gives `"1e+20"`, and +the tie 1000000000000000.25 gives `"1000000000000000.2"`. A FLOAT given +as a string keeps its spelling (Template Schemas §7.5), and a +`LIST[FLOAT]` value given as JSON text reads each number as the nearest +double. Before openjd-model 0.16.0 (openjd-rs#539), a scalar FLOAT +`default` given as a number kept its own digits (`2`, +`100000000000000000000`), a tie rounded up, and JSON text could read a +double one ULP off. White space around a number in a string value is ignored, so `" 7 "` is the INT `7` (openjd-rs#530). A `LIST[INT]`, `LIST[FLOAT]` or `LIST[LIST[INT]]` value given as a Python list of numeric strings is coerced item by item, the same as the JSON text of diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index c44102e8..aae6a9c2 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -210,9 +210,11 @@ environment was entered with (openjd-rs#536). White space around a number in an INT or FLOAT job or task parameter value is ignored, so `" 7 "` is `7` and `" 1.50 "` is `1.50` (openjd-rs#530). -Known gap: a call the session refuses, such as `run_task` on a -`READY_ENDING` session, returns `None` without running anything, and -`action_status` still describes the previous action. The pure-Python +Known gap: a call the session refuses, such as `run_task` or +`enter_environment` on a `READY_ENDING` session, runs nothing and +raises nothing (`enter_environment` still returns an identifier, which +is never entered), and `action_status` still describes the previous +action. The pure-Python reference raises. `clear_ending_only()` is not bound yet. ### SessionUser (stays in Python, passed to Rust) diff --git a/test/openjd/expr/test_strings.py b/test/openjd/expr/test_strings.py index f50d69f5..07fc0a97 100644 --- a/test/openjd/expr/test_strings.py +++ b/test/openjd/expr/test_strings.py @@ -1338,3 +1338,27 @@ def test_cmd_set_variable_roundtrip(self) -> None: # limitation — set requires ^& escaping even inside double quotes. cmd = evaluate_expression(r"'set ' + repr_cmd('MY_VAR=hello world')").item() assert self._cmd_eval(f"{cmd}\necho %MY_VAR%") == "hello world" + + +class TestComputedFloatText: + @pytest.mark.parametrize( + "expr, expected", + [ + pytest.param("string(1000000000000000.25 * 1.0)", "1000000000000000.2", id="scalar"), + pytest.param( + "string([1000000000000000.25 * 1.0])", "[1000000000000000.2]", id="list item" + ), + pytest.param( + "string(float('1000000000000000.25'))", "1000000000000000.2", id="from a string" + ), + ], + ) + def test_a_shortest_digit_tie_rounds_as_repr_does(self, expr: str, expected: str) -> None: + """A computed float is written as ``repr(float)`` writes it (Expression + Language §1.3.4). 1000000000000000.25 lies halfway between the two + shortest candidates, and ``repr`` rounds the tie to even. Since + openjd-expr 0.15.0 (openjd-rs#539) the digits come from ryu, which does + the same; before, Rust's shortest form rounded up, giving + ``1000000000000000.3``.""" + assert repr(1000000000000000.25) == "1000000000000000.2" + assert evaluate_expression(expr).item() == expected diff --git a/test/openjd/model_v1/test_create_job.py b/test/openjd/model_v1/test_create_job.py index 56962a26..4e825457 100644 --- a/test/openjd/model_v1/test_create_job.py +++ b/test/openjd/model_v1/test_create_job.py @@ -3046,11 +3046,16 @@ class TestFloatGivenAsANumber: ``repr(float)`` writes it, with zero always ``0.0`` (Expression Language §1.3.4). A FLOAT given as a string keeps its spelling (Template Schemas §7.5). - What changed in openjd-model 0.16.0 (openjd-rs#539) is a FLOAT parameter's - ``default`` given as a number: before, its own digits were written, so ``2`` - stayed ``2`` and ``1e20`` became ``100000000000000000000``. A value given as - a Python number and a ``LIST[FLOAT]`` default's number items were already - written this way; ``test_value`` and ``test_list_default_items`` pin them. + openjd-model 0.16.0 and openjd-expr 0.15.0 (openjd-rs#539) changed three + things here. A FLOAT parameter's ``default`` given as a number used to keep + its own digits, so ``2`` stayed ``2`` and ``1e20`` became + ``100000000000000000000``. A shortest-digit tie such as + 1000000000000000.25, in a value or a list item, rounded up + (``1000000000000000.3``) where ``repr`` rounds to even. And a LIST[FLOAT] + value given as JSON text could read a double one ULP off. Other values given + as a Python number, and other LIST[FLOAT] default items, were already + written as ``repr`` writes them; ``test_value`` and + ``test_list_default_items`` pin them. """ @pytest.mark.parametrize( @@ -3076,6 +3081,26 @@ def test_list_default_items(self, tmp_path: Path) -> None: defs = [{"name": "L", "type": "LIST[FLOAT]", "default": [2, 1e20]}] assert _preprocess(defs, {}, tmp_path) == {"L": "[2.0, 1e+20]"} + def test_a_shortest_digit_tie_rounds_as_repr_does(self, tmp_path: Path) -> None: + tie = 1000000000000000.2 + assert repr(tie) == "1000000000000000.2" + assert _preprocess([{"name": "F", "type": "FLOAT"}], {"F": tie}, tmp_path) == { + "F": "1000000000000000.2" + } + defs = [{"name": "L", "type": "LIST[FLOAT]", "default": [tie]}] + assert _preprocess(defs, {}, tmp_path) == {"L": "[1000000000000000.2]"} + + def test_list_value_given_as_json_text_reads_the_nearest_double(self, tmp_path: Path) -> None: + """Before, ``2.2250738585072011e-308`` read as ``2.2250738585072014e-308`` + and ``1000000000000000.25`` as ``1000000000000000.4``.""" + text = "[1000000000000000.25, 2.2250738585072011e-308]" + expected = ( + "[" + ", ".join(repr(v) for v in [1000000000000000.25, 2.2250738585072011e-308]) + "]" + ) + assert expected == "[1000000000000000.2, 2.225073858507201e-308]" + defs = [{"name": "L", "type": "LIST[FLOAT]"}] + assert _preprocess(defs, {"L": text}, tmp_path) == {"L": expected} + class TestNumericStringsInListParameters: """openjd-model 0.16.0 (openjd-rs#538). A list value given as a native From 96ccde29fe715242a87355d64f6c5568a4ed93ab Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:44:00 +0000 Subject: [PATCH 13/21] fix: Read LIST[FLOAT] string default items as floats in merged definitions A third review of the bump found that openjd-model 0.16.0 (openjd-rs#538) writes a LIST[FLOAT] default item given as a string, such as "2.50", as a JSON string, so that job creation keeps its spelling. merge_job_parameter_definitions parsed the default as a list of f64, failed, and fell back to returning the raw JSON text as a str, breaking the documented native-type contract: [1.5, "2.50"] came back as '[1.5,"2.50"]' instead of [1.5, 2.5]. Read each item as a number or a numeric string, as JobListFloatParameterDefinition.default does. The same review found more upstream changes visible from Python with no test or note. Pin and document them: - openjd-rs#531: two keyword attributes no longer get placeholders where one is a prefix of the other ("Param.is + Param.assert" gave the wrong value about half the time), and a syntax error shows the text as written ("Param.if +", not "Param.af +"). The expr spec now says unclosed nested f-strings are quadratic, the 200-level bracket cap is gone, and a multi-line expression may end in a comment. - openjd-rs#530: coercing a string to a number with a target_type ignores the white space around it. - openjd-rs#532 also covers exit_environment: a failure before onExit starts fails the action, and the environment counts as exited. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- rust-bindings/src/model/create_job_fns.rs | 17 ++++++++++++++++- specs/python-expr-interface.md | 11 ++++++++--- specs/python-sessions-interface.md | 11 ++++++----- test/openjd/expr/test_parsing.py | 17 +++++++++++++++++ test/openjd/expr/test_strings.py | 19 +++++++++++++++++++ .../model_v1/test_merge_job_parameters.py | 9 +++++++++ 6 files changed, 75 insertions(+), 9 deletions(-) diff --git a/rust-bindings/src/model/create_job_fns.rs b/rust-bindings/src/model/create_job_fns.rs index 1cbea67a..c354a685 100644 --- a/rust-bindings/src/model/create_job_fns.rs +++ b/rust-bindings/src/model/create_job_fns.rs @@ -293,8 +293,23 @@ fn default_to_native( .ok() .map(|v| v.into_py_any(py)) .unwrap_or_else(|| raw.into_py_any(py)), - JobParameterType::ListFloat => serde_json::from_str::>(raw) + // Since openjd-model 0.16.0 (openjd-rs#538) an item written in the + // template as a string, such as `"2.50"`, is a JSON string here so + // that job creation keeps its spelling. A Python `float` can't hold a + // spelling, so read either form as a number, as + // `JobListFloatParameterDefinition.default` does. + JobParameterType::ListFloat => serde_json::from_str::>(raw) .ok() + .and_then(|items| { + items + .iter() + .map(|item| match item { + serde_json::Value::Number(n) => n.as_f64(), + serde_json::Value::String(s) => s.trim().parse::().ok(), + _ => None, + }) + .collect::>>() + }) .map(|v| v.into_py_any(py)) .unwrap_or_else(|| raw.into_py_any(py)), JobParameterType::ListBool => serde_json::from_str::>(raw) diff --git a/specs/python-expr-interface.md b/specs/python-expr-interface.md index a9a11b82..ff634a14 100644 --- a/specs/python-expr-interface.md +++ b/specs/python-expr-interface.md @@ -975,9 +975,14 @@ except RangeExprError as e: Syntax errors use the wording of Ruff's parser (`Expected an expression`, `unexpected EOF while parsing`), which openjd-expr uses since 0.15.0 (openjd-rs#531). Parsing an invalid expression takes time -roughly linear in its length, whatever its bracket nesting. A -multi-line expression is parsed as written, so a caret points into the -text the author wrote. +roughly linear in its length, whatever its bracket nesting, except for +unclosed nested f-strings, which are quadratic. Brackets are no longer +limited to 200 levels; the 64 KB length cap and the AST depth limit +still apply. A multi-line expression is parsed as written, so a caret +points into the text the author wrote, and it may end in a comment. +Coercing a string to a number, for example with a `target_type` of +`int` or `list[int]`, ignores the white space around the number +(openjd-rs#530). A computed float is written as Python's `repr(float)` writes it, including a shortest-digit tie, which rounds to even: diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index aae6a9c2..b37457d8 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -200,11 +200,12 @@ An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session ending-only (`READY_ENDING`) for good: `exit_environment` and `cleanup` still work, and a successful `exit_environment(keep_session_running=True)` leaves it `READY_ENDING` -(openjd-rs#524). When `run_task` or `enter_environment` fails before its -subprocess starts, for example because a parameter value can't be -coerced, the action is `FAILED` with the error's message, as in the -pure-Python reference (openjd-rs#532); an environment that fails this -way still counts as entered. When `exit_environment` can't build its +(openjd-rs#524). When `run_task`, `enter_environment` or +`exit_environment` fails before its subprocess starts, for example +because a parameter value can't be coerced, the action is `FAILED` with +the error's message, as in the pure-Python reference (openjd-rs#532). +An environment whose enter fails this way still counts as entered, and +one whose exit fails this way counts as exited. When `exit_environment` can't build its symbol table from `resolved_symtab`, it runs `onExit` with the table the environment was entered with (openjd-rs#536). White space around a number in an INT or FLOAT job or task parameter value is ignored, so diff --git a/test/openjd/expr/test_parsing.py b/test/openjd/expr/test_parsing.py index 5d394761..362ce597 100644 --- a/test/openjd/expr/test_parsing.py +++ b/test/openjd/expr/test_parsing.py @@ -44,6 +44,23 @@ def test_placeholder_when_every_first_letter_is_taken(self) -> None: result = evaluate_expression(f"[Param.if, Param.xx, len('{names}')]", values=symtab) assert result.item() == [1, 2, 52] + def test_two_keyword_attributes_never_share_a_placeholder(self) -> None: + """Before openjd-expr 0.15.0 (openjd-rs#531) ``is`` and ``assert`` could + be renamed so that one placeholder was a prefix of the other, and this + gave the wrong value about half the time. Repeat it to catch that.""" + symtab = SymbolTable({"Param.is": "a", "Param.assert": "b"}) + for _ in range(50): + assert evaluate_expression("Param.is + Param.assert", values=symtab).item() == "ab" + + def test_a_syntax_error_shows_the_text_as_written(self) -> None: + """Before openjd-expr 0.15.0 (openjd-rs#531) the source line showed the + placeholder (``Param.af +``) instead of the keyword.""" + with pytest.raises(ExpressionError) as exc_info: + evaluate_expression("Param.if +", values=SymbolTable({"Param.if": 1})) + assert str(exc_info.value) == ( + "Syntax error: Expected an expression\n Param.if +\n ^~~~~~~~~~" + ) + def test_keyword_with_conditional(self) -> None: symtab = SymbolTable({"Param.if": 100}) assert evaluate_expression("Param.if if True else 0", values=symtab).item() == 100 diff --git a/test/openjd/expr/test_strings.py b/test/openjd/expr/test_strings.py index 07fc0a97..9792f66b 100644 --- a/test/openjd/expr/test_strings.py +++ b/test/openjd/expr/test_strings.py @@ -1362,3 +1362,22 @@ def test_a_shortest_digit_tie_rounds_as_repr_does(self, expr: str, expected: str ``1000000000000000.3``.""" assert repr(1000000000000000.25) == "1000000000000000.2" assert evaluate_expression(expr).item() == expected + + +class TestNumericStringCoercion: + @pytest.mark.parametrize( + "expr, target, expected", + [ + pytest.param("' 7 '", "int", 7, id="int"), + pytest.param("['1', ' 2']", "list[int]", [1, 2], id="list[int]"), + ], + ) + def test_white_space_around_the_number_is_ignored( + self, expr: str, target: str, expected: object + ) -> None: + """Expression Language §1.2.3: coercing a string to a number ignores the + white space around it. Since openjd-expr 0.15.0 (openjd-rs#530); before, + ``' 7 '`` raised ``Cannot convert ' 7 ' to int``.""" + from openjd.expr import ExprType + + assert evaluate_expression(expr, target_type=ExprType(target)).item() == expected diff --git a/test/openjd/model_v1/test_merge_job_parameters.py b/test/openjd/model_v1/test_merge_job_parameters.py index 1ac4befa..d733ae2b 100644 --- a/test/openjd/model_v1/test_merge_job_parameters.py +++ b/test/openjd/model_v1/test_merge_job_parameters.py @@ -227,6 +227,15 @@ def test_list_float(self) -> None: assert m["default"] == [1.5, 2.5] assert all(isinstance(v, float) for v in m["default"]) + def test_list_float_with_string_items(self) -> None: + """Since openjd-model 0.16.0 (openjd-rs#538) the crate writes a string + item of a LIST[FLOAT] default as a JSON string, to keep its spelling for + job creation. The binding reads it as a ``float``, as the template's + ``default`` does; a ``float`` can't hold the spelling.""" + m = self._merge_one(param_type="LIST[FLOAT]", default=[1.5, "2.50", " 3 "]) + assert m["default"] == [1.5, 2.5, 3.0] + assert all(isinstance(v, float) for v in m["default"]) + def test_list_string(self) -> None: m = self._merge_one(param_type="LIST[STRING]", default=["a", "b"]) assert m["default"] == ["a", "b"] From 01bbde2e5459b2ae1aa6e1a6b4c2f3c6cdeaeb4a Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:01:41 +0000 Subject: [PATCH 14/21] test: Pin a failed exit before onExit starts, and note Windows cleanup retries A fourth review of the bump found the sessions spec's new sentence about exit_environment (openjd-rs#532) had no test. Extend the enter-failure test: the exit fails before onExit starts the same way, and the environment counts as exited. openjd-sessions 0.10.0 (openjd-rs#485) also retries removing a session directory on Windows for up to about 31 s, and the binding's cleanup holds the GIL while it waits. Say so in the sessions spec. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- specs/python-sessions-interface.md | 13 ++++++++----- test/openjd/sessions/test_failed_actions.py | 17 +++++++++++++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index b37457d8..c2259c8d 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -205,11 +205,14 @@ still work, and a successful because a parameter value can't be coerced, the action is `FAILED` with the error's message, as in the pure-Python reference (openjd-rs#532). An environment whose enter fails this way still counts as entered, and -one whose exit fails this way counts as exited. When `exit_environment` can't build its -symbol table from `resolved_symtab`, it runs `onExit` with the table the -environment was entered with (openjd-rs#536). White space around a -number in an INT or FLOAT job or task parameter value is ignored, so -`" 7 "` is `7` and `" 1.50 "` is `1.50` (openjd-rs#530). +one whose exit fails this way counts as exited. When `exit_environment` +can't build its symbol table from `resolved_symtab`, it runs `onExit` +with the table the environment was entered with (openjd-rs#536). White +space around a number in an INT or FLOAT job or task parameter value is +ignored, so `" 7 "` is `7` and `" 1.50 "` is `1.50` (openjd-rs#530). +On Windows, `cleanup` retries removing a session directory that is +still in use, for up to about 31 s, and holds the GIL meanwhile +(openjd-rs#485). Known gap: a call the session refuses, such as `run_task` or `enter_environment` on a `READY_ENDING` session, runs nothing and diff --git a/test/openjd/sessions/test_failed_actions.py b/test/openjd/sessions/test_failed_actions.py index 141726f3..a8281cdf 100644 --- a/test/openjd/sessions/test_failed_actions.py +++ b/test/openjd/sessions/test_failed_actions.py @@ -3,8 +3,8 @@ """How ``Session`` handles a failed action, through openjd-sessions 0.10.0. -- openjd-rs#532: when ``run_task`` or ``enter_environment`` fails before its - subprocess starts, for example because a parameter value can't be coerced, +- openjd-rs#532: when ``run_task``, ``enter_environment`` or + ``exit_environment`` fails before its subprocess starts, for example because a parameter value can't be coerced, the action fails with the error's message and the session becomes ending-only, as the pure-Python ``openjd.sessions`` does. Before, the binding saw an error it discards, so ``action_status`` stayed ``None`` and the @@ -93,14 +93,17 @@ def test_a_task_that_fails_before_it_starts_fails_the_action(tmp_path: Path) -> def test_an_environment_that_fails_before_it_starts_fails_the_action(tmp_path: Path) -> None: - """The environment still counts as entered, so it can be exited.""" + """The environment still counts as entered, so it can be exited, and its + exit fails before ``onExit`` starts the same way and counts as exited.""" session = Session( session_id="s", job_parameter_values={"P": {"type": "PATH", "value": "/a/" + "x" * 32_800}}, session_root_directory=str(tmp_path), ) try: - identifier = session.enter_environment(environment=_environment(onEnter=_python(""))) + identifier = session.enter_environment( + environment=_environment(onEnter=_python(""), onExit=_python("")) + ) _settle(session) status = session.action_status assert status is not None @@ -111,6 +114,12 @@ def test_an_environment_that_fails_before_it_starts_fails_the_action(tmp_path: P ) assert session.state == SessionState.READY_ENDING assert session.environments_entered == [identifier] + session.exit_environment(identifier=identifier) + _settle(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert session.environments_entered == [] finally: session.cleanup() From 98dab674d41e71eeae96a9e23fc683422e3fc961 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:27:59 +0000 Subject: [PATCH 15/21] fix!: Raise the crate's refusal from session actions, and bind clear_ending_only The sessions binding's non-blocking actions (enter_environment, exit_environment, run_task, run_subprocess) handed each call to a background thread and discarded the openjd-sessions crate's Result. A call the crate refused returned as if an action had started, ran nothing, and left the previous action's status showing. With openjd-sessions 0.10.0 (openjd-rs#524) that is the common case after a failed action: the session stays READY_ENDING even after a successful exit_environment(keep_session_running=True), so the next run_task silently did nothing and action_status still read SUCCESS. Each action method now waits, with the GIL released, until the crate either accepts the call or returns. The crate refuses a call before it changes anything, and accepts one by reporting its action through the session callback before the action's first await. A refusal raises SessionError with the crate's message and leaves the state, action_status and environments_entered as they were, as the Rust method returns Err. An accepted call returns once its action is RUNNING, or, for an environment with no script or a failure before the action starts, once the crate has returned, with the session back in its slot. Errors after acceptance are on action_status, where the crate records them; the binding records one there itself if the crate ever doesn't. A call while another action runs gets the crate's InvalidState message instead of "An action is already running". Bind Session.clear_ending_only(), the crate's way back from ending-only to READY. It raises SessionError unless the session is READY_ENDING. WindowsSessionUser no longer treats a failure to look up the process user as "not the process user"; it raises, as the pure-Python reference does. Tests: the strict xfail for a refused run_task becomes test/openjd/sessions/test_refused_calls.py, which drives each refusal through the binding and checks the exception, the crate's message, and that state, action_status and environments_entered didn't change, plus clear_ending_only and accepted calls. The sessions spec, the quality report and the stub's Session class (regenerated, which also adds run_task's step_name) describe the new behavior. BREAKING CHANGE: Session.enter_environment, exit_environment, run_task and run_subprocess raise SessionError for a call the crate refuses, including on a READY_ENDING session, instead of returning. A call while another action runs raises SessionError with the crate's InvalidState message rather than RuntimeError("An action is already running"); SessionError is a RuntimeError subclass. WindowsSessionUser raises if the process user can't be determined. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- ...ions-bindings-quality-evaluation-report.md | 4 +- rust-bindings/src/sessions/session.rs | 424 ++++++++++++------ rust-bindings/src/sessions/session_user.rs | 14 +- specs/python-sessions-interface.md | 40 +- src/openjd/_openjd_rs.pyi | 48 +- test/openjd/sessions/test_failed_actions.py | 2 +- test/openjd/sessions/test_known_gaps.py | 62 --- test/openjd/sessions/test_refused_calls.py | 361 +++++++++++++++ 8 files changed, 724 insertions(+), 231 deletions(-) delete mode 100644 test/openjd/sessions/test_known_gaps.py create mode 100644 test/openjd/sessions/test_refused_calls.py diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index 3bc26be8..a19a3537 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -369,8 +369,8 @@ counted as test-suite failures. | Failure | Reference | Binding | Status | |---|---|---|---| -| `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `RuntimeError("An action is already running")` | ⚠ message change | -| `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | runs nothing (`run_task` returns `None`; `enter_environment` returns an identifier that was never entered), `action_status` still shows the previous action (the binding discards the crate's `Err`); openjd-sessions 0.10.0 keeps a session `READY_ENDING` after a failed action even when an exit keeps it running | ❌ silent no-op (strict xfail in `test/openjd/sessions/test_known_gaps.py`) | +| `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `SessionError("Session must be in READY state, current: RUNNING")` (the crate's message; a `RuntimeError`) | ⚠ message change | +| `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | `SessionError("Session must be in READY state, current: READY_ENDING")`, a `RuntimeError`; nothing changes | ✓ resolved (raises the crate's refusal; `clear_ending_only()` bound) | | `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change | | `cancel_action` while running | succeeds | `RuntimeError("Cannot cancel: session is busy with an action")` | ❌ contract inversion | | `Session(session_root_directory=…)` non-existent dir | `RuntimeError("Ensure that the root directory ...")` | likely Rust I/O error → `SessionError(...)` (untested) | ⚠ untested | diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index c7d95f3d..5b520ef1 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -12,7 +12,9 @@ use pyo3_stub_gen::derive::*; use openjd_model::types::{JobParameterType, JobParameterValue, JobParameterValues}; use openjd_model::types::{TaskParameterSet, TaskParameterType, TaskParameterValue}; +use openjd_sessions::action::ActionState; use openjd_sessions::action_status::ActionStatus; +use openjd_sessions::error::SessionError; use openjd_sessions::limits::SessionLimits; use openjd_sessions::session::{Session, SessionConfig, SessionState}; @@ -158,6 +160,40 @@ fn failed_action_status(message: &str) -> ActionStatus { } } +/// What an action thread tells the Python thread that handed it a call. +/// +/// A non-blocking method must not return until it knows whether the crate +/// accepted the call. The crate refuses a call (wrong state, unknown or +/// duplicate environment, out-of-order exit, bad `run_subprocess` +/// arguments) before it touches the session, and such a refusal has to +/// raise. It accepts a call by reporting the call's action through the +/// session callback, before its first `.await`; from then on every outcome, +/// errors included, is recorded on the action status. +enum CallOutcome { + /// The crate accepted the call and its action is `Running`. + Started, + /// The crate returned before the action reported `Running`. + /// `accepted == false` means it refused the call and changed nothing, so + /// `error` is the refusal. Otherwise the action ended without running a + /// subprocess (an environment with no script, or a failure before the + /// action started), and its status says how. + Returned { + accepted: bool, + error: Option, + }, +} + +/// The call an action thread is running, until the Python thread that made +/// it has been told the outcome. +struct PendingCall { + tx: std::sync::mpsc::SyncSender, + /// Set by the session callback: the crate reported an action for this + /// call, so it accepted it. + accepted: bool, +} + +type PendingSlot = Arc>>; + /// Body of a spawned action thread. /// /// Builds a lightweight current-thread Tokio runtime, runs the action @@ -166,7 +202,7 @@ fn failed_action_status(message: &str) -> ActionStatus { /// the runtime failed to build or the future panicked. This guarantees: /// /// * the session is never left permanently "taken" (which would wedge -/// every later call into `"An action is already running"`), and +/// every later call), and /// * a panic in async session code can't escape the detached thread to /// poison shared state silently and tear down the interpreter's view /// of the session. @@ -176,13 +212,18 @@ fn failed_action_status(message: &str) -> ActionStatus { /// inside the action is likewise converted into a terminal FAILED /// `ActionStatus` (rather than copying the possibly-still-Running real /// session state), so the poller never hangs on a panicked action. +/// +/// If the calling Python thread is still waiting (the action never +/// reported `Running`), it is told the outcome after the session is back in +/// its slot, so it can make its next call straight away. fn run_action( mut session: Session, session_arc: Arc>>, snapshot: Arc>, + pending: PendingSlot, action: F, ) where - F: FnOnce(&tokio::runtime::Runtime, &mut Session), + F: FnOnce(&tokio::runtime::Runtime, &mut Session) -> Result<(), SessionError>, { let rt = match tokio::runtime::Builder::new_current_thread() .enable_all() @@ -193,6 +234,7 @@ fn run_action( // Couldn't build a runtime (e.g. fd/thread exhaustion). Mark // the action FAILED and drop out of RUNNING so the poller // delivers a terminal callback instead of hanging. + let call = lock_recover(&pending).take(); { let mut snap = lock_recover(&snapshot); snap.state = SessionState::Ready; @@ -201,22 +243,26 @@ fn run_action( ))); } *lock_recover(&session_arc) = Some(session); + tell(call, true, None); return; } }; // Catch panics so a panic inside async session code still restores // the session below rather than leaking out of this detached thread. - let action_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - action(&rt, &mut session); - })); + let action_result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| action(&rt, &mut session))); + + // Take the waiting call, if any, before the session goes back: once it + // is back another thread can start a call and install its own. + let call = lock_recover(&pending).take(); + let accepted = call.as_ref().is_none_or(|c| c.accepted); // Restore the session BEFORE refreshing the snapshot, holding the guard // across both so there's no window where a caller finds the slot empty. // Python callers watch `state` via the snapshot and dispatch the next // action the moment it transitions out of Running; if the snapshot - // updated first they could find the mutex still empty ("An action is - // already running"). + // updated first they could find the mutex still empty. let guard = { let mut slot = lock_recover(&session_arc); *slot = Some(session); @@ -224,10 +270,17 @@ fn run_action( }; match action_result { - Ok(()) => { + Ok(result) => { if let Some(s) = guard.as_ref() { PySession::update_snapshot(s, &snapshot); } + drop(guard); + if accepted { + if let Err(e) = &result { + record_unreported_failure(&snapshot, e); + } + } + tell(call, accepted, result.err()); } Err(payload) => { // The action panicked mid-run. `catch_unwind` kept the session @@ -236,16 +289,47 @@ fn run_action( // would leave the Python poller waiting forever. Force a // terminal FAILED status and drop out of RUNNING, mirroring the // runtime-build-failure branch above. - let mut snap = lock_recover(&snapshot); - snap.state = SessionState::Ready; - snap.action_status = Some(failed_action_status(&format!( - "action panicked during execution: {}", - panic_detail(payload.as_ref()) - ))); + drop(guard); + { + let mut snap = lock_recover(&snapshot); + snap.state = SessionState::Ready; + snap.action_status = Some(failed_action_status(&format!( + "action panicked during execution: {}", + panic_detail(payload.as_ref()) + ))); + } + tell(call, true, None); } } } +/// Tell a waiting Python thread how its call came out. Nothing to do when +/// it was already told the action started. +fn tell(call: Option, accepted: bool, error: Option) { + if let Some(call) = call { + // `send` fails only when the receiver is gone, and the caller drops + // it only after receiving: there is nobody left to tell. + let _ = call.tx.send(CallOutcome::Returned { accepted, error }); + } +} + +/// An accepted call returned an error. The crate records every such error +/// on the action status (`fail_action_setup`, `drive_action`, or a failed +/// script's own status), so this normally changes nothing. If the status +/// doesn't show a failure, record the error there rather than lose it. +fn record_unreported_failure(snapshot: &Arc>, e: &SessionError) { + let mut snap = lock_recover(snapshot); + let reported = snap.action_status.as_ref().is_some_and(|st| { + matches!( + st.state, + ActionState::Failed | ActionState::Canceled | ActionState::Timeout + ) + }); + if !reported { + snap.action_status = Some(failed_action_status(&e.to_string())); + } +} + /// Best-effort extraction of a human-readable message from a caught panic /// payload (`std::panic::catch_unwind`'s `Err` value), which is usually the /// `&str` or `String` passed to `panic!`. @@ -271,6 +355,9 @@ pub(crate) struct PySession { /// by a background action thread — the case where `Session::cancel_action` /// is unreachable (it needs `&mut Session`). cancel_handle: openjd_sessions::session::SessionCancelHandle, + /// The call whose caller is waiting to learn whether the crate accepted + /// it. Shared with the session callback, which marks it accepted. + pending: PendingSlot, } impl PySession { @@ -280,6 +367,68 @@ impl PySession { snap.action_status = session.action_status(); snap.environments_entered = session.environments_entered().to_vec(); } + + /// The error for a call made while an action thread owns the session. + /// The crate itself would refuse it with `InvalidState`, because the + /// session is `Running` or `Canceling`; give the same message. Until the + /// action reports `Running` the snapshot may still show the state + /// before it, so anything else reads as `Running`. + fn busy_error(&self, expected: &[SessionState]) -> PyErr { + let current = match lock_recover(&self.snapshot).state { + s @ (SessionState::Running | SessionState::Canceling) => s, + _ => SessionState::Running, + }; + session_err_to_py(SessionError::InvalidState { + expected: expected.to_vec(), + current, + }) + } + + /// Run `action` on a background thread and return once the crate has + /// accepted it and its action is `Running`, or once it has returned. + /// A call the crate refuses raises `SessionError` with the crate's + /// message and leaves the session, its state and its action status as + /// they were. Every later outcome of an accepted call is reported on the + /// action status. The GIL is released while waiting, because the action + /// thread needs it to log. + fn dispatch(&self, py: Python<'_>, expected: &[SessionState], action: F) -> PyResult<()> + where + F: FnOnce(&tokio::runtime::Runtime, &mut Session) -> Result<(), SessionError> + + Send + + 'static, + { + let session = { + let mut guard = lock_recover(&self.session); + match guard.take() { + Some(session) => session, + None => { + drop(guard); + return Err(self.busy_error(expected)); + } + } + }; + let (tx, rx) = std::sync::mpsc::sync_channel(1); + *lock_recover(&self.pending) = Some(PendingCall { + tx, + accepted: false, + }); + + let session_arc = self.session.clone(); + let snapshot = self.snapshot.clone(); + let pending = self.pending.clone(); + std::thread::spawn(move || run_action(session, session_arc, snapshot, pending, action)); + + match py.detach(move || rx.recv()) { + Ok(CallOutcome::Returned { + accepted: false, + error: Some(e), + }) => Err(session_err_to_py(e)), + // Started, an accepted call that returned (its status says how), + // or an action thread that ended without saying (it panicked + // outside `catch_unwind`; the snapshot reports the failure). + _ => Ok(()), + } + } } #[cfg_attr(feature = "stub-gen", gen_stub_pymethods)] @@ -345,9 +494,27 @@ impl PySession { environments_entered: vec![], })); let snapshot_for_cb = snapshot.clone(); + let pending: PendingSlot = Arc::new(Mutex::new(None)); + let pending_for_cb = pending.clone(); + // The crate calls this for every action it accepts before the + // action's first `.await`, so the first call for a pending call is + // its acceptance. On `Running`, the caller is told the action + // started; `state` is set here, under the snapshot lock and before + // the action thread's final `update_snapshot`, so it can't + // overwrite a later state. Lock order: snapshot, then pending. let callback = Box::new(move |_session_id: &str, status: &ActionStatus| { - if let Ok(mut snap) = snapshot_for_cb.lock() { - snap.action_status = Some(status.clone()); + let mut snap = lock_recover(&snapshot_for_cb); + snap.action_status = Some(status.clone()); + let mut slot = lock_recover(&pending_for_cb); + if let Some(call) = slot.as_mut() { + call.accepted = true; + if status.state == ActionState::Running { + snap.state = SessionState::Running; + if let Some(call) = slot.take() { + // Fails only if the caller is gone; see `tell`. + let _ = call.tx.send(CallOutcome::Started); + } + } } }); @@ -382,6 +549,7 @@ impl PySession { session: Arc::new(Mutex::new(Some(session))), snapshot, cancel_handle, + pending, }) } @@ -453,12 +621,28 @@ impl PySession { } // ── Non-blocking action methods ── - - /// Enter an environment. Non-blocking — spawns the onEnter action on a - /// background thread and returns the environment identifier immediately. + // + // Each returns once its action is `Running` (or, for an action that + // runs no subprocess, once it has ended) and leaves the rest to a + // background thread. A call the crate refuses raises `SessionError` + // with the crate's message, as the Rust method returns `Err`, and + // changes nothing; that includes a call while another action is + // running or the session is ending-only. Once the crate has accepted a + // call, its outcome, including an error before the action's subprocess + // starts, is on `action_status`. + + /// Enter an environment. Non-blocking: returns the environment + /// identifier once the onEnter action is running. + /// + /// Raises `SessionError` if the session isn't `READY`, if + /// `identifier` is already entered, or if a second environment with a + /// wrap hook is entered. A failure after that, such as an environment + /// variable that doesn't resolve, fails the action; the environment + /// still counts as entered and must be exited. #[pyo3(signature = (*, environment, identifier=None, resolved_symtab=None, os_env_vars=None))] fn enter_environment( &self, + py: Python<'_>, environment: &PyEnvironment, identifier: Option, resolved_symtab: Option<&crate::expr::PySerializedSymbolTable>, @@ -482,86 +666,56 @@ impl PySession { // evaluation or expression interpolation. let resolved = resolved_symtab.map(|st| st.inner.clone()); - // Take the session out of the mutex so the background thread owns it - let mut guard = lock_recover(&self.session); - let session = guard.take().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("An action is already running") + self.dispatch(py, &[SessionState::Ready], move |rt, session| { + rt.block_on(session.enter_environment( + &env, + resolved.as_ref(), + Some(&env_id), + os_env_vars.as_ref(), + )) + .map(|_| ()) })?; - drop(guard); - - // Update snapshot to RUNNING and CLEAR action_status. - // The worker's poll loop treats a non-None action_status as "the current - // action has reported state". If we leave the previous action's status - // here, the poll loop will immediately fire a stale terminal callback - // for the new action — causing the worker to think envEnter / run_task - // finished in ~1ms with the previous action's SUCCEEDED result. - { - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Running; - snap.action_status = None; - } - - let session_arc = self.session.clone(); - let snapshot = self.snapshot.clone(); - - std::thread::spawn(move || { - run_action(session, session_arc, snapshot, move |rt, session| { - let env_ref = os_env_vars.as_ref(); - let _ = rt.block_on(session.enter_environment( - &env, - resolved.as_ref(), - Some(&env_id), - env_ref, - )); - }); - }); - Ok(return_id) } - /// Exit an environment. Non-blocking — spawns the onExit action on a - /// background thread. + /// Exit an environment. Non-blocking: returns once the onExit action is + /// running. + /// + /// Raises `SessionError` if the session isn't `READY` or + /// `READY_ENDING`, if `identifier` isn't entered, or if it isn't the + /// most recently entered environment. A failure after that fails the + /// action, and the environment still counts as exited. #[pyo3(signature = (*, identifier, resolved_symtab=None, keep_session_running=true, os_env_vars=None))] fn exit_environment( &self, + py: Python<'_>, identifier: String, resolved_symtab: Option<&crate::expr::PySerializedSymbolTable>, keep_session_running: bool, os_env_vars: Option>, ) -> PyResult<()> { let resolved = resolved_symtab.map(|st| st.inner.clone()); - - let mut guard = lock_recover(&self.session); - let session = guard.take().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("An action is already running") - })?; - drop(guard); - - { - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Running; - snap.action_status = None; - } - - let session_arc = self.session.clone(); - let snapshot = self.snapshot.clone(); - - std::thread::spawn(move || { - run_action(session, session_arc, snapshot, move |rt, session| { - let env_ref = os_env_vars.as_ref(); - let _ = rt.block_on(session.exit_environment( + self.dispatch( + py, + &[SessionState::Ready, SessionState::ReadyEnding], + move |rt, session| { + rt.block_on(session.exit_environment( &identifier, resolved.as_ref(), keep_session_running, - env_ref, - )); - }); - }); - - Ok(()) + os_env_vars.as_ref(), + )) + .map(|_| ()) + }, + ) } - /// Run a task. Non-blocking — spawns the onRun action on a background thread. + /// Run a task. Non-blocking: returns once the onRun action is running. + /// + /// Raises `SessionError` if the session isn't `READY`, for example + /// because an earlier action failed and the session is ending-only. A + /// failure after that, such as a task parameter value that can't be + /// coerced, fails the action. /// /// `step_name` is surfaced as `WrappedStep.Name` to a wrapping /// environment's `onWrapTaskRun` hook (RFC 0008). It defaults to an empty @@ -569,6 +723,7 @@ impl PySession { #[pyo3(signature = (*, step_script, step_name=None, task_parameter_values=None, resolved_symtab=None, os_env_vars=None))] fn run_task( &self, + py: Python<'_>, step_script: &PyStepScript, step_name: Option, task_parameter_values: Option<&Bound<'_, PyDict>>, @@ -583,42 +738,27 @@ impl PySession { }; let resolved = resolved_symtab.map(|st| st.inner.clone()); - let mut guard = lock_recover(&self.session); - let session = guard.take().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("An action is already running") - })?; - drop(guard); - - { - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Running; - snap.action_status = None; - } - - let session_arc = self.session.clone(); - let snapshot = self.snapshot.clone(); - - std::thread::spawn(move || { - run_action(session, session_arc, snapshot, move |rt, session| { - let env_ref = os_env_vars.as_ref(); - let task_ref = task_params.as_ref(); - let _ = rt.block_on(session.run_task( - &step_name, - &script, - task_ref, - resolved.as_ref(), - env_ref, - )); - }); - }); - - Ok(()) + self.dispatch(py, &[SessionState::Ready], move |rt, session| { + rt.block_on(session.run_task( + &step_name, + &script, + task_params.as_ref(), + resolved.as_ref(), + os_env_vars.as_ref(), + )) + .map(|_| ()) + }) } - /// Run a subprocess. Non-blocking — spawns on a background thread. + /// Run a subprocess. Non-blocking: returns once it is running. + /// + /// Raises `SessionError` if the session isn't `READY`, if `command` is + /// empty or blank, or if `timeout` is zero. #[pyo3(signature = (*, command, args=None, timeout=None, os_env_vars=None, use_session_env_vars=true, log_banner_message=None))] + #[allow(clippy::too_many_arguments)] // PyO3 method mirrors a kwarg-rich public signature fn run_subprocess( &self, + py: Python<'_>, command: String, args: Option>, timeout: Option, @@ -626,38 +766,36 @@ impl PySession { use_session_env_vars: bool, log_banner_message: Option, ) -> PyResult<()> { - let mut guard = lock_recover(&self.session); - let session = guard.take().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err("An action is already running") - })?; - drop(guard); - - { - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Running; - snap.action_status = None; - } - - let session_arc = self.session.clone(); - let snapshot = self.snapshot.clone(); - - std::thread::spawn(move || { - run_action(session, session_arc, snapshot, move |rt, session| { - let duration = timeout.map(std::time::Duration::from_secs_f64); - let env_ref = os_env_vars.as_ref(); - let args_ref = args.as_deref(); - let banner_ref = log_banner_message.as_deref(); - let _ = rt.block_on(session.run_subprocess( - &command, - args_ref, - duration, - env_ref, - use_session_env_vars, - banner_ref, - )); - }); - }); + let duration = timeout.map(std::time::Duration::from_secs_f64); + self.dispatch(py, &[SessionState::Ready], move |rt, session| { + rt.block_on(session.run_subprocess( + &command, + args.as_deref(), + duration, + os_env_vars.as_ref(), + use_session_env_vars, + log_banner_message.as_deref(), + )) + .map(|_| ()) + }) + } + /// Take an ending-only session (`READY_ENDING`) back to `READY`, so it + /// accepts `run_task`, `enter_environment` and `run_subprocess` again. + /// The environments entered, the last action's status and the + /// environment variables stay as they were. + /// + /// Raises `SessionError` if the session isn't `READY_ENDING`. The + /// crate also refuses after its session-wide cancel token is canceled; + /// this binding doesn't expose one, so that can't happen here. + fn clear_ending_only(&self) -> PyResult<()> { + let mut guard = lock_recover(&self.session); + let Some(session) = guard.as_mut() else { + drop(guard); + return Err(self.busy_error(&[SessionState::ReadyEnding])); + }; + session.clear_ending_only().map_err(session_err_to_py)?; + Self::update_snapshot(session, &self.snapshot); Ok(()) } diff --git a/rust-bindings/src/sessions/session_user.rs b/rust-bindings/src/sessions/session_user.rs index e1441f39..8aab7b4d 100644 --- a/rust-bindings/src/sessions/session_user.rs +++ b/rust-bindings/src/sessions/session_user.rs @@ -200,10 +200,16 @@ impl PyWindowsSessionUser { )); } - // Determine whether `user` matches the process user. - let is_process_user = openjd_sessions::win32::get_process_user() - .map(|proc_user| user.eq_ignore_ascii_case(&proc_user)) - .unwrap_or(false); + // Determine whether `user` matches the process user. A failure to + // look up the process user raises, as the pure-Python reference's + // `get_process_user()` does, rather than reading as "not the + // process user". + let process_user = openjd_sessions::win32::get_process_user().map_err(|e| { + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Could not determine the process user: {e}" + )) + })?; + let is_process_user = user.eq_ignore_ascii_case(&process_user); if is_process_user { if password.is_some() { diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index c2259c8d..853e6f21 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -117,7 +117,7 @@ class Session: files_directory: Path action_status: Optional[ActionStatus] - # Environment lifecycle (non-blocking — starts action, returns immediately) + # Environment lifecycle (non-blocking — returns once the action is running) def enter_environment( self, *, @@ -133,7 +133,7 @@ class Session: os_env_vars: Optional[dict[str, str]] = None, ) -> None: ... - # Task execution (non-blocking — starts action, returns immediately) + # Task execution (non-blocking — returns once the action is running) def run_task( self, *, @@ -162,6 +162,8 @@ class Session: mark_action_failed: bool = False, ) -> None: ... + def clear_ending_only(self) -> None: ... + def cleanup(self) -> None: ... ``` @@ -196,11 +198,30 @@ resolves it. is 8 random characters from `[a-z0-9_]`, with no part of the session ID in it, as the pure-Python reference names it. +A call the openjd-sessions crate refuses raises `SessionError` with the +crate's message, as the Rust method returns `Err`, and changes nothing: +the state, `action_status` (still the previous action's) and +`environments_entered` stay as they were. The crate refuses `run_task`, +`enter_environment` and `run_subprocess` unless the session is `READY`, +`exit_environment` unless it is `READY` or `READY_ENDING`, an unknown, +out-of-order or duplicate environment identifier, a second environment +with wrap hooks, and a blank `command` or zero `timeout`. A call made +while another action runs gets the same `InvalidState` message +(`Session must be in READY state, current: RUNNING`); before, it was +`RuntimeError("An action is already running")`, and `SessionError` is a +`RuntimeError`. Once the crate accepts a call, the method returns when +its action is `RUNNING` (or, for an environment with no script, has +ended), and `action_status` describes that action. Everything after +acceptance, errors included, is reported on `action_status`; the method +doesn't raise for it, where the Rust method would also return `Err`, +because it has already returned. + An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session -ending-only (`READY_ENDING`) for good: `exit_environment` and `cleanup` -still work, and a successful -`exit_environment(keep_session_running=True)` leaves it `READY_ENDING` -(openjd-rs#524). When `run_task`, `enter_environment` or +ending-only (`READY_ENDING`): `exit_environment` and `cleanup` still +work, and a successful `exit_environment(keep_session_running=True)` +leaves it `READY_ENDING` (openjd-rs#524). `clear_ending_only()` takes it +back to `READY`, keeping the environments entered and the last action's +status; it raises `SessionError` unless the session is `READY_ENDING`. When `run_task`, `enter_environment` or `exit_environment` fails before its subprocess starts, for example because a parameter value can't be coerced, the action is `FAILED` with the error's message, as in the pure-Python reference (openjd-rs#532). @@ -214,13 +235,6 @@ On Windows, `cleanup` retries removing a session directory that is still in use, for up to about 31 s, and holds the GIL meanwhile (openjd-rs#485). -Known gap: a call the session refuses, such as `run_task` or -`enter_environment` on a `READY_ENDING` session, runs nothing and -raises nothing (`enter_environment` still returns an identifier, which -is never entered), and `action_status` still describes the previous -action. The pure-Python -reference raises. `clear_ending_only()` is not bound yet. - ### SessionUser (stays in Python, passed to Rust) The `SessionUser` types involve OS-specific credential validation (sudo diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index f877d908..85984a7c 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -2213,8 +2213,14 @@ class Session: os_env_vars: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None, ) -> builtins.str: r""" - Enter an environment. Non-blocking — spawns the onEnter action on a - background thread and returns the environment identifier immediately. + Enter an environment. Non-blocking: returns the environment + identifier once the onEnter action is running. + + Raises `SessionError` if the session isn't `READY`, if + `identifier` is already entered, or if a second environment with a + wrap hook is entered. A failure after that, such as an environment + variable that doesn't resolve, fails the action; the environment + still counts as entered and must be exited. """ def exit_environment( @@ -2226,20 +2232,35 @@ class Session: os_env_vars: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None, ) -> None: r""" - Exit an environment. Non-blocking — spawns the onExit action on a - background thread. + Exit an environment. Non-blocking: returns once the onExit action is + running. + + Raises `SessionError` if the session isn't `READY` or + `READY_ENDING`, if `identifier` isn't entered, or if it isn't the + most recently entered environment. A failure after that fails the + action, and the environment still counts as exited. """ def run_task( self, *, step_script: StepScript, + step_name: typing.Optional[builtins.str] = None, task_parameter_values: typing.Optional[dict] = None, resolved_symtab: typing.Optional[SerializedSymbolTable] = None, os_env_vars: typing.Optional[typing.Mapping[builtins.str, builtins.str]] = None, ) -> None: r""" - Run a task. Non-blocking — spawns the onRun action on a background thread. + Run a task. Non-blocking: returns once the onRun action is running. + + Raises `SessionError` if the session isn't `READY`, for example + because an earlier action failed and the session is ending-only. A + failure after that, such as a task parameter value that can't be + coerced, fails the action. + + `step_name` is surfaced as `WrappedStep.Name` to a wrapping + environment's `onWrapTaskRun` hook (RFC 0008). It defaults to an empty + string when the caller does not supply one. """ def run_subprocess( @@ -2253,7 +2274,22 @@ class Session: log_banner_message: typing.Optional[builtins.str] = None, ) -> None: r""" - Run a subprocess. Non-blocking — spawns on a background thread. + Run a subprocess. Non-blocking: returns once it is running. + + Raises `SessionError` if the session isn't `READY`, if `command` is + empty or blank, or if `timeout` is zero. + """ + + def clear_ending_only(self) -> None: + r""" + Take an ending-only session (`READY_ENDING`) back to `READY`, so it + accepts `run_task`, `enter_environment` and `run_subprocess` again. + The environments entered, the last action's status and the + environment variables stay as they were. + + Raises `SessionError` if the session isn't `READY_ENDING`. The + crate also refuses after its session-wide cancel token is canceled; + this binding doesn't expose one, so that can't happen here. """ def cancel_action( diff --git a/test/openjd/sessions/test_failed_actions.py b/test/openjd/sessions/test_failed_actions.py index a8281cdf..a0303f93 100644 --- a/test/openjd/sessions/test_failed_actions.py +++ b/test/openjd/sessions/test_failed_actions.py @@ -19,7 +19,7 @@ value is ignored. Before, such a value failed before the action started, which the binding didn't report, so the task silently didn't run. -What a session does with a call it then refuses is in ``test_known_gaps.py``. +What a session does with a call it refuses is in ``test_refused_calls.py``. """ from __future__ import annotations diff --git a/test/openjd/sessions/test_known_gaps.py b/test/openjd/sessions/test_known_gaps.py deleted file mode 100644 index 005717f2..00000000 --- a/test/openjd/sessions/test_known_gaps.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Known gaps between the sessions binding and the pure-Python reference. - -Each test is a strict xfail. When a gap is fixed, move its test next to the -surface it covers rather than leaving it here. -""" - -from __future__ import annotations - -import sys -import time -from pathlib import Path - -import pytest - -from openjd._openjd_rs import Session, SessionState, deserialize_step - -# ── A call the session refuses returns as if it had started an action ── -# -# The non-blocking actions in ``rust-bindings/src/sessions/session.rs`` -# (``run_task``, ``enter_environment``, ``exit_environment``, -# ``run_subprocess``) hand the call to a background thread and discard the -# crate's ``Result``. A call the crate refuses up front, such as ``run_task`` -# on an ending-only session, therefore returns ``None``, runs nothing, and -# leaves ``action_status`` showing the previous action. The pure-Python -# ``openjd.sessions.Session`` raises for a session that isn't ``READY``. -# -# The gap predates openjd-sessions 0.10.0: a ``run_task`` straight after a -# failed one, as below, was refused the same way with 0.9.0. 0.10.0 -# (openjd-rs#524) adds a common path into it: a failed action now keeps the -# session ending-only even after a successful -# ``exit_environment(keep_session_running=True)``, so the next ``run_task`` is -# refused and the caller sees the exit's ``SUCCESS``. With 0.9.0 that exit -# returned the session to ``READY`` and the ``run_task`` ran. - - -@pytest.mark.xfail( - reason="A run_task the crate refuses returns None and leaves the previous action_status", - strict=True, -) -def test_run_task_on_an_ending_only_session_raises(tmp_path: Path) -> None: - def script(code: str): - return deserialize_step( - { - "name": "S", - "script": {"actions": {"onRun": {"command": sys.executable, "args": ["-c", code]}}}, - } - ).script - - session = Session(session_id="s", job_parameter_values={}, session_root_directory=str(tmp_path)) - try: - session.run_task(step_script=script("raise SystemExit(1)")) - deadline = time.monotonic() + 30 - while session.state == SessionState.RUNNING and time.monotonic() < deadline: - time.sleep(0.05) - assert session.state == SessionState.READY_ENDING - with pytest.raises(Exception): - session.run_task(step_script=script("")) - finally: - session.cleanup() diff --git a/test/openjd/sessions/test_refused_calls.py b/test/openjd/sessions/test_refused_calls.py new file mode 100644 index 00000000..cedc9ff8 --- /dev/null +++ b/test/openjd/sessions/test_refused_calls.py @@ -0,0 +1,361 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A call the openjd-sessions crate refuses raises, as the Rust method returns ``Err``. + +The crate refuses a call before it changes anything: the session state, the +last action's status and the environments entered stay as they were. Each test +drives a sequence through the binding and checks the crate's documented result: +the exception class, its message (the crate's ``SessionError`` text), the state, +and that ``action_status`` still describes the previous action. + +Before, the binding handed every call to a background thread and discarded the +crate's ``Result``, so a refused call returned as if an action had started, ran +nothing, and left the previous action's status showing. openjd-sessions 0.10.0 +(openjd-rs#524) made that common: a failed action keeps the session ending-only +even after a successful ``exit_environment(keep_session_running=True)``. + +``clear_ending_only`` is the crate's way out of ending-only (openjd-rs#524). +""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path +from typing import Any, Callable, Optional + +import pytest + +from openjd import _openjd_rs +from openjd._openjd_rs import ( + ActionState, + ActionStatus, + Session, + SessionState, + create_environment, + decode_environment_template, + deserialize_step, +) + +# Registered at runtime by `register_renamed_exception`; the type stub doesn't +# declare the exception classes. +SessionError: type[Exception] = getattr(_openjd_rs, "SessionError") + + +def _python(code: str) -> dict[str, Any]: + return {"command": sys.executable, "args": ["-c", code]} + + +def _step_script(code: str) -> Any: + return deserialize_step({"name": "S", "script": {"actions": {"onRun": _python(code)}}}).script + + +def _environment(name: str = "E", *, extensions: Optional[list[str]] = None, **actions: Any) -> Any: + template: dict[str, Any] = { + "specificationVersion": "environment-2023-09", + "environment": {"name": name, "script": {"actions": actions}}, + } + if extensions: + template["extensions"] = extensions + return create_environment( + decode_environment_template(template=template, supported_extensions=extensions) + ) + + +def _settle(session: Session) -> None: + deadline = time.monotonic() + 30 + while session.state == SessionState.RUNNING and time.monotonic() < deadline: + time.sleep(0.02) + assert session.state != SessionState.RUNNING, "action did not finish within 30s" + + +def _status(session: Session) -> Optional[tuple]: + status: Optional[ActionStatus] = session.action_status + if status is None: + return None + return (status.state, status.exit_code, status.fail_message, status.status_message) + + +@pytest.fixture +def session(tmp_path: Path): + s = Session(session_id="s", job_parameter_values={}, session_root_directory=str(tmp_path)) + try: + yield s + finally: + s.cleanup() + + +def _assert_refused(session: Session, call: Callable[[], object], message: str) -> None: + """``call`` raises ``SessionError(message)`` and changes nothing.""" + state = session.state + status = _status(session) + entered = session.environments_entered + with pytest.raises(SessionError) as exc_info: + call() + assert str(exc_info.value) == message + assert session.state == state + assert _status(session) == status + assert session.environments_entered == entered + + +def _make_ending_only(session: Session) -> str: + """Enter an environment, fail a task in it, and exit it with + ``keep_session_running=True``. Returns the identifier of a second, outer + environment that is still entered.""" + outer = session.enter_environment(environment=_environment("Outer", onEnter=_python(""))) + _settle(session) + inner = session.enter_environment(environment=_environment("Inner", onEnter=_python(""))) + _settle(session) + session.run_task(step_script=_step_script("raise SystemExit(1)")) + _settle(session) + session.exit_environment(identifier=inner, keep_session_running=True) + _settle(session) + status = session.action_status + assert status is not None and status.state == ActionState.SUCCESS + assert session.state == SessionState.READY_ENDING + return outer + + +class TestOnAnEndingOnlySession: + """openjd-rs#524: an ending-only session accepts only ``exit_environment`` + and ``cleanup``. The others raise ``InvalidState``.""" + + _MESSAGE = "Session must be in READY state, current: READY_ENDING" + + def test_run_task(self, session: Session, tmp_path: Path) -> None: + marker = tmp_path / "ran" + _make_ending_only(session) + _assert_refused( + session, + lambda: session.run_task( + step_script=_step_script(f"open({str(marker)!r}, 'w').close()") + ), + self._MESSAGE, + ) + time.sleep(0.2) + assert not marker.exists() + + def test_enter_environment(self, session: Session) -> None: + _make_ending_only(session) + _assert_refused( + session, + lambda: session.enter_environment(environment=_environment(onEnter=_python(""))), + self._MESSAGE, + ) + + def test_run_subprocess(self, session: Session) -> None: + _make_ending_only(session) + _assert_refused( + session, + lambda: session.run_subprocess(command=sys.executable, args=["-c", ""]), + self._MESSAGE, + ) + + def test_exit_environment_is_still_accepted(self, session: Session) -> None: + outer = _make_ending_only(session) + session.exit_environment(identifier=outer) + _settle(session) + status = session.action_status + assert status is not None and status.state == ActionState.SUCCESS + assert session.environments_entered == [] + assert session.state == SessionState.READY_ENDING + + +class TestClearEndingOnly: + def test_returns_the_session_to_ready(self, session: Session) -> None: + """The last action's status and the environments entered stay as + they were, and the session accepts new actions again.""" + outer = _make_ending_only(session) + status = _status(session) + session.clear_ending_only() + assert session.state == SessionState.READY + assert _status(session) == status + assert session.environments_entered == [outer] + session.run_task(step_script=_step_script("")) + _settle(session) + new_status = session.action_status + assert new_status is not None and new_status.state == ActionState.SUCCESS + assert session.state == SessionState.READY + + def test_a_failed_action_makes_it_ending_only_again(self, session: Session) -> None: + _make_ending_only(session) + session.clear_ending_only() + session.run_task(step_script=_step_script("raise SystemExit(1)")) + _settle(session) + assert session.state == SessionState.READY_ENDING + + def test_refused_when_the_session_is_ready(self, session: Session) -> None: + _assert_refused( + session, + session.clear_ending_only, + "Session must be in READY_ENDING state, current: READY", + ) + + def test_refused_while_an_action_runs(self, session: Session) -> None: + session.run_task(step_script=_step_script("import time; time.sleep(1)")) + try: + assert session.state == SessionState.RUNNING + with pytest.raises(SessionError) as exc_info: + session.clear_ending_only() + assert str(exc_info.value) == ( + "Session must be in READY_ENDING state, current: RUNNING" + ) + finally: + # Let the task finish rather than cancel it: the crate reports + # the action `Running` just before it installs the action's + # cancel token, so a cancel straight after the call returns can + # find no action to cancel. + _settle(session) + + +class TestWhileAnActionRuns: + """The crate refuses every action while one is running. Before, the + binding raised ``RuntimeError("An action is already running")``; it now + gives the crate's ``InvalidState`` message. ``SessionError`` is a + ``RuntimeError``, so a caller catching ``RuntimeError`` still does.""" + + @pytest.mark.parametrize( + "call, message", + [ + pytest.param( + lambda s: s.run_task(step_script=_step_script("")), + "Session must be in READY state, current: RUNNING", + id="run_task", + ), + pytest.param( + lambda s: s.enter_environment(environment=_environment(onEnter=_python(""))), + "Session must be in READY state, current: RUNNING", + id="enter_environment", + ), + pytest.param( + lambda s: s.exit_environment(identifier="E"), + "Session must be in READY or READY_ENDING state, current: RUNNING", + id="exit_environment", + ), + pytest.param( + lambda s: s.run_subprocess(command=sys.executable, args=["-c", ""]), + "Session must be in READY state, current: RUNNING", + id="run_subprocess", + ), + ], + ) + def test_a_second_action( + self, session: Session, call: Callable[[Session], object], message: str + ) -> None: + session.run_task(step_script=_step_script("import time; time.sleep(1)")) + try: + assert session.state == SessionState.RUNNING + running = _status(session) + with pytest.raises(SessionError) as exc_info: + call(session) + assert isinstance(exc_info.value, RuntimeError) + assert str(exc_info.value) == message + assert session.state == SessionState.RUNNING + assert _status(session) == running + finally: + # Let the task finish rather than cancel it: the crate reports + # the action `Running` just before it installs the action's + # cancel token, so a cancel straight after the call returns can + # find no action to cancel. + _settle(session) + + +class TestOtherRefusals: + def test_exit_an_unknown_environment(self, session: Session) -> None: + _assert_refused( + session, + lambda: session.exit_environment(identifier="nope"), + "Unknown environment identifier: nope", + ) + + def test_exit_out_of_order(self, session: Session) -> None: + first = session.enter_environment(environment=_environment("A", onEnter=_python(""))) + _settle(session) + second = session.enter_environment(environment=_environment("B", onEnter=_python(""))) + _settle(session) + _assert_refused( + session, + lambda: session.exit_environment(identifier=first), + f"Must exit the most recently entered environment first. Expected {second}, got {first}", + ) + + def test_enter_a_duplicate_identifier(self, session: Session) -> None: + session.enter_environment(environment=_environment(onEnter=_python("")), identifier="A") + _settle(session) + _assert_refused( + session, + lambda: session.enter_environment( + environment=_environment(onEnter=_python("")), identifier="A" + ), + "Environment A has already been entered in this Session.", + ) + + def test_enter_a_second_wrap_environment(self, session: Session) -> None: + def wrap_environment(name: str) -> Any: + hooks = ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit") + return _environment( + name, extensions=["EXPR", "WRAP_ACTIONS"], **{h: _python("") for h in hooks} + ) + + session.enter_environment(environment=wrap_environment("W1")) + _settle(session) + _assert_refused( + session, + lambda: session.enter_environment(environment=wrap_environment("W2")), + "RFC 0008: a session may have at most one Environment defining wrap hooks " + "(onWrapEnvEnter / onWrapTaskRun / onWrapEnvExit). Environment 'W1' already " + "defines wrap hooks; cannot also enter 'W2'.", + ) + + @pytest.mark.parametrize( + "kwargs, message", + [ + pytest.param({"command": " "}, "command must be a non-empty string", id="blank"), + pytest.param( + {"command": sys.executable, "timeout": 0.0}, + "timeout must be positive", + id="zero timeout", + ), + ], + ) + def test_run_subprocess_arguments( + self, session: Session, kwargs: dict[str, Any], message: str + ) -> None: + _assert_refused(session, lambda: session.run_subprocess(**kwargs), message) + + +class TestAcceptedCalls: + """An accepted call returns once its action is running, with + ``action_status`` describing the new action, not the previous one.""" + + def test_run_task_reports_the_new_action(self, session: Session) -> None: + session.run_task(step_script=_step_script("raise SystemExit(1)")) + _settle(session) + session.clear_ending_only() + session.run_task(step_script=_step_script("import time; time.sleep(0.5)")) + status = session.action_status + assert status is not None + assert status.state == ActionState.RUNNING + assert session.state == SessionState.RUNNING + _settle(session) + status = session.action_status + assert status is not None and status.state == ActionState.SUCCESS + + def test_an_environment_with_no_script_has_ended_on_return(self, session: Session) -> None: + """Nothing runs, so the action has ended by the time the call + returns, and the session is back in its slot for the next call.""" + environment = create_environment( + decode_environment_template( + template={ + "specificationVersion": "environment-2023-09", + "environment": {"name": "E", "variables": {"X": "1"}}, + } + ) + ) + identifier = session.enter_environment(environment=environment) + status = session.action_status + assert status is not None and status.state == ActionState.SUCCESS + assert session.state == SessionState.READY + session.exit_environment(identifier=identifier) + assert session.environments_entered == [] From 90b8090cc52ffff0374aafceac219a9c857fbab8 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:17:37 +0000 Subject: [PATCH 16/21] fix: Cancel straight after an action starts, and close the binding's other gaps A review of the previous commit found: - A cancel_action made as soon as run_task, run_subprocess or enter_environment returned nearly always raised "Cannot cancel: no action is running", and the action ran to the end. The crate reports an action Running just before it installs the action's cancel token, and the call now returns on that report. cancel_action now keeps trying, with the GIL released, until the token is there or the action thread has given the session back, and then lets the crate decide, so "nothing to cancel" is always the crate's InvalidState message. It sets the snapshot's Canceling state under the session lock, so the snapshot can't be left reading Canceling after the action has ended. - A negative, NaN or infinite run_subprocess timeout or cancel_action time_limit panicked in Duration::from_secs_f64, now in the method body, raising PanicException, which except Exception doesn't catch. It now raises ValueError. Zero still reaches the crate, which refuses it. - If the action runtime couldn't be built, the call returned as if accepted and the snapshot was overwritten. It is now a refusal: the session is put back unchanged and the call raises SessionError. If the action thread can't be spawned, the session is put back rather than lost. - The fallback that records an unreported error on action_status ran after the session was back in its slot, where a next action could already have started. It now runs under the session lock. - cleanup() held the GIL through the crate's Windows removal retries (up to about 31 s). It now releases it. cancel_action's arguments now default to None, as the spec documents. Pin each change, and correct the sessions spec (errors after acceptance don't raise even when they arrive before the call returns) and the quality report's rows for cancel_action, run_subprocess and the environment refusals. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- ...ions-bindings-quality-evaluation-report.md | 21 ++- rust-bindings/src/sessions/session.rs | 166 ++++++++++++------ specs/python-sessions-interface.md | 32 ++-- src/openjd/_openjd_rs.pyi | 26 ++- test/openjd/sessions/test_refused_calls.py | 88 ++++++++-- 5 files changed, 243 insertions(+), 90 deletions(-) diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index a19a3537..f44e9015 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -346,12 +346,12 @@ counted as test-suite failures. | `ActionResult` | not in reference | new pyclass, Python-constructible via `#[new]`, pickleable | ✓ resolved (Rec #11) | | `Session.__init__(callback=...)` | accepted | accepted only by wrapper, NOT by binding | ⚠ binding gap | | `Session.__init__(revision_extensions=...)` | accepted, used | accepted by wrapper, **not forwarded** | ❌ silent regression | -| `Session.cancel_action()` | cancels a running action | raises `RuntimeError("session is busy")` while running | ❌ correctness regression | +| `Session.cancel_action()` | cancels a running action | cancels it, also straight after the call that started it | ✓ resolved | | `Session.__enter__/__exit__` | implemented | not implemented on wrapper | ❌ missing | | `Session.session_id` (property) | not exposed in ref either; spec says yes | not on wrapper; on binding only | ⚠ matches ref | | `Session.run_subprocess(timeout=0)` | raises `ValueError` | silently runs without timeout (wrapper: `if timeout else None`) | ❌ correctness regression | -| `Session.run_subprocess(timeout=-5)` | raises `ValueError` | panics in background thread; action wedges | ❌ correctness regression | -| `Session.run_subprocess(command="")` | raises `ValueError` | accepts; action wedges with `action_status=None` | ❌ correctness regression | +| `Session.run_subprocess(timeout=-5)` | raises `ValueError` | binding raises `ValueError` | ✓ resolved | +| `Session.run_subprocess(command="")` | raises `ValueError` | binding raises `SessionError("command must be a non-empty string")`, the crate's refusal | ⚠ class change | | `Session.run_task(task_parameter_values)` | required (`TaskParameterSet`) | wrapper makes it required, but accepts dicts of dicts | ⚠ silent type drift | | `Session.exit_environment(keep_session_running=False)` | default `False` | wrapper default `True` | ❌ silent default flip | | `Session.environments_entered` (property) | `tuple[str, ...]` | binding returns `list[str]`; wrapper returns `tuple` | ✓ wrapper matches | @@ -371,11 +371,11 @@ counted as test-suite failures. |---|---|---|---| | `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `SessionError("Session must be in READY state, current: RUNNING")` (the crate's message; a `RuntimeError`) | ⚠ message change | | `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | `SessionError("Session must be in READY state, current: READY_ENDING")`, a `RuntimeError`; nothing changes | ✓ resolved (raises the crate's refusal; `clear_ending_only()` bound) | -| `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change | -| `cancel_action` while running | succeeds | `RuntimeError("Cannot cancel: session is busy with an action")` | ❌ contract inversion | +| `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change (the crate's) | +| `cancel_action` while running | succeeds | succeeds, including straight after the call that started the action | ✓ resolved | | `Session(session_root_directory=…)` non-existent dir | `RuntimeError("Ensure that the root directory ...")` | likely Rust I/O error → `SessionError(...)` (untested) | ⚠ untested | -| `enter_environment` with same identifier twice | `RuntimeError("Environment {id} has already been entered ...")` | not verified | ⚠ untested | -| `exit_environment` with unknown id | `RuntimeError("Cannot exit unknown Environment ...")` | not verified | ⚠ untested | +| `enter_environment` with same identifier twice | `RuntimeError("Environment {id} has already been entered ...")` | `SessionError` with the same message | ✓ tested | +| `exit_environment` with unknown id | `RuntimeError("Cannot exit unknown Environment ...")` | `SessionError("Unknown environment identifier: {id}")` | ⚠ message change (the crate's; tested) | ### 5.3 Underlying Rust crate — `openjd-rs` @@ -472,6 +472,9 @@ All 17 are reproducible failures of the binding/wrapper. `test_known_gaps::test_cancel_action_actually_cancels_running_action`. Reference fix path: thread the Rust crate's `CancellationToken` to a `cancel()` method that does not need `&mut Session`. **High priority.** + **Resolved** — the binding cancels through the crate's + `SessionCancelHandle`, and retries for the moment between the crate + reporting `Running` and installing the action's cancel token. 2. **`run_subprocess` skips all input validation.** * `timeout=0` silently disables (wrapper bug at `_session.py` line in @@ -482,6 +485,10 @@ All 17 are reproducible failures of the binding/wrapper. * `command=""` — same wedge: the action sits at RUNNING with `action_status=None`. Reference's reference behavior is `ValueError` for all three. **High priority.** + **Partly resolved** — the binding raises `SessionError` with the crate's + refusal for a blank `command` and a zero `timeout` (given to the binding), + and `ValueError` for a negative, NaN or infinite `timeout`. The `_v1` + wrapper's `if timeout else None` is fixed in openjd-sessions-for-python. 3. **State enums lost `.value`.** `SessionState.READY.value` → `AttributeError`. The new `bindings-rs` test_session_scenarios.py uses `state.value` and is diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index 5b520ef1..bcc5809b 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -160,6 +160,17 @@ fn failed_action_status(message: &str) -> ActionStatus { } } +/// Convert a timeout or time limit given in seconds. Zero passes through, so +/// the crate gives its own refusal for it. A negative, NaN or infinite value +/// has no `Duration`, so it raises `ValueError` here rather than panicking. +fn seconds_to_duration(name: &str, seconds: f64) -> PyResult { + std::time::Duration::try_from_secs_f64(seconds).map_err(|_| { + pyo3::exceptions::PyValueError::new_err(format!( + "{name} must be a finite, non-negative number of seconds, got {seconds}" + )) + }) +} + /// What an action thread tells the Python thread that handed it a call. /// /// A non-blocking method must not return until it knows whether the crate @@ -231,19 +242,18 @@ fn run_action( { Ok(rt) => rt, Err(e) => { - // Couldn't build a runtime (e.g. fd/thread exhaustion). Mark - // the action FAILED and drop out of RUNNING so the poller - // delivers a terminal callback instead of hanging. + // Couldn't build a runtime (e.g. fd/thread exhaustion). The + // crate never saw the call, so treat it as refused: put the + // session back unchanged and raise in the caller. let call = lock_recover(&pending).take(); - { - let mut snap = lock_recover(&snapshot); - snap.state = SessionState::Ready; - snap.action_status = Some(failed_action_status(&format!( - "failed to start action runtime: {e}" - ))); - } *lock_recover(&session_arc) = Some(session); - tell(call, true, None); + tell( + call, + false, + Some(SessionError::Runtime(format!( + "failed to start action runtime: {e}" + ))), + ); return; } }; @@ -274,12 +284,14 @@ fn run_action( if let Some(s) = guard.as_ref() { PySession::update_snapshot(s, &snapshot); } - drop(guard); + // Still under the session guard, so no next action can have + // started and been overwritten. if accepted { if let Err(e) = &result { record_unreported_failure(&snapshot, e); } } + drop(guard); tell(call, accepted, result.err()); } Err(payload) => { @@ -354,7 +366,7 @@ pub(crate) struct PySession { /// to whichever action is running even while the `Session` value is owned /// by a background action thread — the case where `Session::cancel_action` /// is unreachable (it needs `&mut Session`). - cancel_handle: openjd_sessions::session::SessionCancelHandle, + cancel_handle: Arc, /// The call whose caller is waiting to learn whether the crate accepted /// it. Shared with the session callback, which marks it accepted. pending: PendingSlot, @@ -413,10 +425,31 @@ impl PySession { accepted: false, }); + // `run_action` owns the session from here. If the thread can't be + // spawned, the closure is dropped unrun, so keep the session where + // it can be recovered, put it back, and raise, as for a refusal. + let session_slot = Arc::new(Mutex::new(Some(session))); + let for_thread = session_slot.clone(); let session_arc = self.session.clone(); let snapshot = self.snapshot.clone(); let pending = self.pending.clone(); - std::thread::spawn(move || run_action(session, session_arc, snapshot, pending, action)); + let spawned = std::thread::Builder::new() + .name("openjd-session-action".into()) + .spawn(move || { + let Some(session) = lock_recover(&for_thread).take() else { + return; + }; + run_action(session, session_arc, snapshot, pending, action) + }); + if let Err(e) = spawned { + *lock_recover(&self.pending) = None; + if let Some(session) = lock_recover(&session_slot).take() { + *lock_recover(&self.session) = Some(session); + } + return Err(session_err_to_py(SessionError::Runtime(format!( + "failed to start the action thread: {e}" + )))); + } match py.detach(move || rx.recv()) { Ok(CallOutcome::Returned { @@ -544,7 +577,7 @@ impl PySession { snap.files_directory = session.files_directory().to_string_lossy().to_string(); snap.action_status = session.action_status(); } - let cancel_handle = session.cancel_handle(); + let cancel_handle = Arc::new(session.cancel_handle()); Ok(PySession { session: Arc::new(Mutex::new(Some(session))), snapshot, @@ -753,7 +786,8 @@ impl PySession { /// Run a subprocess. Non-blocking: returns once it is running. /// /// Raises `SessionError` if the session isn't `READY`, if `command` is - /// empty or blank, or if `timeout` is zero. + /// empty or blank, or if `timeout` is zero, and `ValueError` if + /// `timeout` is negative, NaN or infinite. #[pyo3(signature = (*, command, args=None, timeout=None, os_env_vars=None, use_session_env_vars=true, log_banner_message=None))] #[allow(clippy::too_many_arguments)] // PyO3 method mirrors a kwarg-rich public signature fn run_subprocess( @@ -766,7 +800,9 @@ impl PySession { use_session_env_vars: bool, log_banner_message: Option, ) -> PyResult<()> { - let duration = timeout.map(std::time::Duration::from_secs_f64); + let duration = timeout + .map(|t| seconds_to_duration("timeout", t)) + .transpose()?; self.dispatch(py, &[SessionState::Ready], move |rt, session| { rt.block_on(session.run_subprocess( &command, @@ -799,50 +835,74 @@ impl PySession { Ok(()) } + /// Cancel the running action. + /// + /// Raises `SessionError` with the crate's message if no action is + /// running, and `ValueError` if `time_limit` is negative, NaN or + /// infinite. + #[pyo3(signature = (time_limit=None, mark_action_failed=None))] fn cancel_action( &self, + py: Python<'_>, time_limit: Option, mark_action_failed: Option, ) -> PyResult<()> { - let duration = time_limit.map(std::time::Duration::from_secs_f64); + let duration = time_limit + .map(|t| seconds_to_duration("time_limit", t)) + .transpose()?; let mark_failed = mark_action_failed.unwrap_or(false); - // Direct path when the session is not taken (e.g. between actions): - // Session::cancel_action also performs the Running -> Canceling state - // transition, which the handle cannot (it doesn't own the Session). - let mut guard = lock_recover(&self.session); - if let Some(session) = guard.as_mut() { - let result = session - .cancel_action(duration, mark_failed) - .map_err(session_err_to_py); - Self::update_snapshot(session, &self.snapshot); - return result; - } - drop(guard); - - // Session is owned by a background action thread: deliver through the - // thread-safe cancel handle. Cancellation follows the action's own - // cancelation method (including cross-user helper delivery), exactly - // like Session::cancel_action. - if self.cancel_handle.cancel(duration, mark_failed) { - // The handle cannot set the session's transient Canceling state; - // reflect it in the snapshot so Python-side pollers observe the - // cancel-in-progress phase until the terminal state lands. - let mut snap = lock_recover(&self.snapshot); - snap.state = SessionState::Canceling; - Ok(()) - } else { - Err(pyo3::exceptions::PyRuntimeError::new_err( - "Cannot cancel: no action is running", - )) - } + let session_arc = self.session.clone(); + let snapshot = self.snapshot.clone(); + let handle = self.cancel_handle.clone(); + // The crate reports an action `Running` just before it installs the + // action's cancel token, so a cancel straight after the call that + // started it can find no token yet. Keep trying until the token is + // there or the action thread has given the session back; the gap is + // well under a millisecond. The GIL is released meanwhile. + py.detach(move || loop { + { + let mut guard = lock_recover(&session_arc); + if let Some(session) = guard.as_mut() { + // Between actions, or the action has ended: the crate + // decides, and refuses with `InvalidState` unless the + // session is `Running` (which it isn't here). + let result = session + .cancel_action(duration, mark_failed) + .map_err(session_err_to_py); + Self::update_snapshot(session, &snapshot); + return result; + } + // Session is owned by a background action thread: deliver + // through the thread-safe cancel handle, which follows the + // action's own cancelation method (including cross-user + // helper delivery), exactly like Session::cancel_action. + // Holding the session lock means the action thread can't + // write its final snapshot before the `Canceling` below, so + // the snapshot can't be left reading `Canceling` after the + // action has ended. + if handle.cancel(duration, mark_failed) { + lock_recover(&snapshot).state = SessionState::Canceling; + return Ok(()); + } + } + std::thread::sleep(std::time::Duration::from_micros(100)); + }) } - fn cleanup(&self) { - let mut guard = lock_recover(&self.session); - if let Some(session) = guard.as_mut() { - session.cleanup(); - Self::update_snapshot(session, &self.snapshot); - } + /// Clean up the session, deleting its working directory unless it is + /// retained. Does nothing while an action is running. The GIL is + /// released meanwhile: on Windows the crate retries a removal that + /// fails for up to about 31 s. + fn cleanup(&self, py: Python<'_>) { + let session_arc = self.session.clone(); + let snapshot = self.snapshot.clone(); + py.detach(move || { + let mut guard = lock_recover(&session_arc); + if let Some(session) = guard.as_mut() { + session.cleanup(); + Self::update_snapshot(session, &snapshot); + } + }); } fn __repr__(&self, py: Python<'_>) -> PyResult { diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index 853e6f21..648d6a12 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -142,7 +142,7 @@ class Session: os_env_vars: Optional[dict[str, str]] = None, ) -> None: ... - # Subprocess execution (non-blocking) + # Subprocess execution (non-blocking — returns once it is running) def run_subprocess( self, *, @@ -203,18 +203,24 @@ crate's message, as the Rust method returns `Err`, and changes nothing: the state, `action_status` (still the previous action's) and `environments_entered` stay as they were. The crate refuses `run_task`, `enter_environment` and `run_subprocess` unless the session is `READY`, -`exit_environment` unless it is `READY` or `READY_ENDING`, an unknown, -out-of-order or duplicate environment identifier, a second environment -with wrap hooks, and a blank `command` or zero `timeout`. A call made -while another action runs gets the same `InvalidState` message -(`Session must be in READY state, current: RUNNING`); before, it was -`RuntimeError("An action is already running")`, and `SessionError` is a -`RuntimeError`. Once the crate accepts a call, the method returns when -its action is `RUNNING` (or, for an environment with no script, has -ended), and `action_status` describes that action. Everything after -acceptance, errors included, is reported on `action_status`; the method -doesn't raise for it, where the Rust method would also return `Err`, -because it has already returned. +`exit_environment` unless it is `READY` or `READY_ENDING`, +`cancel_action` unless an action is running, an unknown, out-of-order +or duplicate environment identifier, a second environment with wrap +hooks, and a blank `command` or zero `timeout`. A call made while +another action runs gets the crate's `InvalidState` message, such as +`Session must be in READY state, current: RUNNING`; before, it was +`RuntimeError("An action is already running")`, and `SessionError` is +a `RuntimeError`. A `timeout` or `time_limit` that is negative, NaN or +infinite raises `ValueError`. + +Once the crate accepts a call, the method returns when its action is +`RUNNING` (or, for an environment with no script, has ended), and +`action_status` describes that action. A `cancel_action` made as soon as +it returns reaches the action. Every error after acceptance, such as a +parameter value that can't be coerced, is reported on `action_status` +and doesn't raise, even when it arrives before the call returns. The +crate documents those as failing the action rather than refusing the +call, and the pure-Python reference reports them the same way. An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session ending-only (`READY_ENDING`): `exit_environment` and `cleanup` still diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index 85984a7c..6d722dfa 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -2277,7 +2277,8 @@ class Session: Run a subprocess. Non-blocking: returns once it is running. Raises `SessionError` if the session isn't `READY`, if `command` is - empty or blank, or if `timeout` is zero. + empty or blank, or if `timeout` is zero, and `ValueError` if + `timeout` is negative, NaN or infinite. """ def clear_ending_only(self) -> None: @@ -2294,10 +2295,25 @@ class Session: def cancel_action( self, - time_limit: typing.Optional[builtins.float], - mark_action_failed: typing.Optional[builtins.bool], - ) -> None: ... - def cleanup(self) -> None: ... + time_limit: typing.Optional[builtins.float] = None, + mark_action_failed: typing.Optional[builtins.bool] = None, + ) -> None: + r""" + Cancel the running action. + + Raises `SessionError` with the crate's message if no action is + running, and `ValueError` if `time_limit` is negative, NaN or + infinite. + """ + + def cleanup(self) -> None: + r""" + Clean up the session, deleting its working directory unless it is + retained. Does nothing while an action is running. The GIL is + released meanwhile: on Windows the crate retries a removal that + fails for up to about 31 s. + """ + def __repr__(self) -> builtins.str: ... @typing.final diff --git a/test/openjd/sessions/test_refused_calls.py b/test/openjd/sessions/test_refused_calls.py index cedc9ff8..dd0760bb 100644 --- a/test/openjd/sessions/test_refused_calls.py +++ b/test/openjd/sessions/test_refused_calls.py @@ -65,9 +65,10 @@ def _environment(name: str = "E", *, extensions: Optional[list[str]] = None, **a def _settle(session: Session) -> None: deadline = time.monotonic() + 30 - while session.state == SessionState.RUNNING and time.monotonic() < deadline: + busy = (SessionState.RUNNING, SessionState.CANCELING) + while session.state in busy and time.monotonic() < deadline: time.sleep(0.02) - assert session.state != SessionState.RUNNING, "action did not finish within 30s" + assert session.state not in busy, "action did not finish within 30s" def _status(session: Session) -> Optional[tuple]: @@ -193,7 +194,7 @@ def test_refused_when_the_session_is_ready(self, session: Session) -> None: ) def test_refused_while_an_action_runs(self, session: Session) -> None: - session.run_task(step_script=_step_script("import time; time.sleep(1)")) + session.run_task(step_script=_step_script("import time; time.sleep(30)")) try: assert session.state == SessionState.RUNNING with pytest.raises(SessionError) as exc_info: @@ -202,10 +203,7 @@ def test_refused_while_an_action_runs(self, session: Session) -> None: "Session must be in READY_ENDING state, current: RUNNING" ) finally: - # Let the task finish rather than cancel it: the crate reports - # the action `Running` just before it installs the action's - # cancel token, so a cancel straight after the call returns can - # find no action to cancel. + session.cancel_action() _settle(session) @@ -243,7 +241,7 @@ class TestWhileAnActionRuns: def test_a_second_action( self, session: Session, call: Callable[[Session], object], message: str ) -> None: - session.run_task(step_script=_step_script("import time; time.sleep(1)")) + session.run_task(step_script=_step_script("import time; time.sleep(30)")) try: assert session.state == SessionState.RUNNING running = _status(session) @@ -254,10 +252,7 @@ def test_a_second_action( assert session.state == SessionState.RUNNING assert _status(session) == running finally: - # Let the task finish rather than cancel it: the crate reports - # the action `Running` just before it installs the action's - # cancel token, so a cancel straight after the call returns can - # find no action to cancel. + session.cancel_action() _settle(session) @@ -324,6 +319,75 @@ def test_run_subprocess_arguments( ) -> None: _assert_refused(session, lambda: session.run_subprocess(**kwargs), message) + @pytest.mark.parametrize( + "timeout, shown", [(-1.0, "-1"), (float("nan"), "NaN"), (float("inf"), "inf")] + ) + def test_run_subprocess_timeout_with_no_duration( + self, session: Session, timeout: float, shown: str + ) -> None: + """A timeout with no ``Duration`` raises ``ValueError`` before the + crate sees it. Before, the binding's conversion panicked + (``PanicException``, which ``except Exception`` doesn't catch).""" + with pytest.raises(ValueError) as exc_info: + session.run_subprocess(command=sys.executable, timeout=timeout) + assert str(exc_info.value) == ( + f"timeout must be a finite, non-negative number of seconds, got {shown}" + ) + assert session.state == SessionState.READY + assert session.action_status is None + + +_SLEEP = "import time; time.sleep(30)" + + +class TestCancelAction: + def test_with_no_action_running(self, session: Session) -> None: + """The crate's ``InvalidState``, whether or not an action has run.""" + message = "Session must be in RUNNING state, current: READY" + _assert_refused(session, session.cancel_action, message) + session.run_task(step_script=_step_script("")) + _settle(session) + _assert_refused(session, session.cancel_action, message) + + def test_a_time_limit_with_no_duration(self, session: Session) -> None: + with pytest.raises(ValueError) as exc_info: + session.cancel_action(time_limit=-1.0) + assert str(exc_info.value) == ( + "time_limit must be a finite, non-negative number of seconds, got -1" + ) + + @pytest.mark.parametrize( + "start", + [ + pytest.param(lambda s: s.run_task(step_script=_step_script(_SLEEP)), id="run_task"), + pytest.param( + lambda s: s.run_subprocess(command=sys.executable, args=["-c", _SLEEP]), + id="run_subprocess", + ), + pytest.param( + lambda s: s.enter_environment(environment=_environment(onEnter=_python(_SLEEP))), + id="enter_environment", + ), + ], + ) + def test_straight_after_the_call_that_started_the_action( + self, session: Session, start: Callable[[Session], object] + ) -> None: + """The crate reports an action ``Running`` just before it installs + its cancel token. A cancel made as soon as the call returns still + reaches the action; before, it nearly always raised "Cannot cancel: + no action is running" and the action ran to the end.""" + for _ in range(5): + start(session) + began = time.monotonic() + session.cancel_action() + _settle(session) + assert time.monotonic() - began < 20 + status = session.action_status + assert status is not None and status.state == ActionState.CANCELED + assert session.state == SessionState.READY_ENDING + session.clear_ending_only() + class TestAcceptedCalls: """An accepted call returns once its action is running, with From 8077e875b462a27d6a386841bafe14d653f70579 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:29:20 +0000 Subject: [PATCH 17/21] fix: Never wait for the session lock while holding the GIL A second review found that cleanup(), which now releases the GIL, could deadlock with a concurrent call when INFO logging was on. cleanup holds the session lock while the crate logs, and pyo3_log takes the GIL for each record, while the other thread held the GIL waiting for the same lock (dispatch, clear_ending_only, extend_path_mapping_rules). Every lock of the session is now taken with the GIL released. A test runs the pattern in a fresh interpreter with INFO logging. Also from the review: - cancel_action's retry for a missing cancel token could spin for ever when called on the action thread itself, from a logging handler on the crate's "Running Task" banner, which the crate logs before installing the token. The wait is now bounded (2 s), then it raises SessionError "Cannot cancel: the action has not started yet". Pinned in a fresh interpreter. - The panic branch of the action thread wrote its forced FAILED status after releasing the session lock, where a next action could already have started. It now writes it under the lock. - The sessions spec said cleanup holds the GIL, and its signature block lagged the binding; it now matches the stub, and says a cancel straight after an action that has already ended raises InvalidState and a zero time_limit terminates at once. The quality report's recommendations for cancel and the negative timeout are struck through, the timeout=0 rows are updated, and clear_ending_only is listed as binding-only API. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- ...ions-bindings-quality-evaluation-report.md | 23 +++-- rust-bindings/src/sessions/session.rs | 86 +++++++++++------- specs/python-sessions-interface.md | 14 ++- test/openjd/sessions/test_refused_calls.py | 88 +++++++++++++++++++ 4 files changed, 167 insertions(+), 44 deletions(-) diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index f44e9015..6bc7892f 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -240,7 +240,7 @@ a Python class in `_v1/_session.py` that holds a `_RustSession` and adds: |---|---|---|---| | `Session.__init__(revision_extensions=...)` | accepted **and used** | accepted, never forwarded | `get_enabled_extensions()` always returns `[]` | | `exit_environment(keep_session_running=...)` | `False` | `True` | Default flip — silent semantic regression for callers who omit the kwarg | -| `run_subprocess(timeout=...)` | int>0; ValueError on 0/None handling | `float(timeout) if timeout else None` | `timeout=0` silently disables the timeout | +| `run_subprocess(timeout=...)` | int>0; ValueError on 0/None handling | `None if timeout is None else float(timeout)` | ✓ resolved: `timeout=0` reaches the crate and raises | | `run_task(task_parameter_values: TaskParameterSet)` | required positional / kw | required kw — but type-coerces dicts and ParameterValue instances inconsistently | Possible bad-value paths | ### 3.4 Missing wrapper API @@ -349,7 +349,7 @@ counted as test-suite failures. | `Session.cancel_action()` | cancels a running action | cancels it, also straight after the call that started it | ✓ resolved | | `Session.__enter__/__exit__` | implemented | not implemented on wrapper | ❌ missing | | `Session.session_id` (property) | not exposed in ref either; spec says yes | not on wrapper; on binding only | ⚠ matches ref | -| `Session.run_subprocess(timeout=0)` | raises `ValueError` | silently runs without timeout (wrapper: `if timeout else None`) | ❌ correctness regression | +| `Session.run_subprocess(timeout=0)` | raises `ValueError` | raises `SessionError("timeout must be positive")`, the crate's refusal | ⚠ class change | | `Session.run_subprocess(timeout=-5)` | raises `ValueError` | binding raises `ValueError` | ✓ resolved | | `Session.run_subprocess(command="")` | raises `ValueError` | binding raises `SessionError("command must be a non-empty string")`, the crate's refusal | ⚠ class change | | `Session.run_task(task_parameter_values)` | required (`TaskParameterSet`) | wrapper makes it required, but accepts dicts of dicts | ⚠ silent type drift | @@ -370,7 +370,8 @@ counted as test-suite failures. | Failure | Reference | Binding | Status | |---|---|---|---| | `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `SessionError("Session must be in READY state, current: RUNNING")` (the crate's message; a `RuntimeError`) | ⚠ message change | -| `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | `SessionError("Session must be in READY state, current: READY_ENDING")`, a `RuntimeError`; nothing changes | ✓ resolved (raises the crate's refusal; `clear_ending_only()` bound) | +| `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | `SessionError("Session must be in READY state, current: READY_ENDING")`, a `RuntimeError`; nothing changes | ⚠ message change (the crate's) | +| `Session.clear_ending_only()` | not in reference | takes a `READY_ENDING` session back to `READY`, as the crate's `clear_ending_only` does | ⚠ binding-only API | | `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change (the crate's) | | `cancel_action` while running | succeeds | succeeds, including straight after the call that started the action | ✓ resolved | | `Session(session_root_directory=…)` non-existent dir | `RuntimeError("Ensure that the root directory ...")` | likely Rust I/O error → `SessionError(...)` (untested) | ⚠ untested | @@ -542,12 +543,14 @@ Numbered for the report-driven workflow in `~/openjd-rs/AGENTS.md`. Strike throu ### Priority 1 — correctness regressions (fix before any caller migration) -1. Make `Session.cancel_action()` work while an action is running. Wire the +1. ~~Make `Session.cancel_action()` work while an action is running. Wire the Rust crate's `CancellationToken` through to a binding-owned handle so cancel does not require `&mut Session`. Reference test: `test/openjd/sessions-v1/test_known_gaps.py::test_cancel_action_actually_cancels_running_action`. Files: `rust-bindings/src/sessions/session.rs` lines 446–459; openjd-rs - `openjd-sessions/src/session.rs` cancel surface. + `openjd-sessions/src/session.rs` cancel surface.~~ **Resolved** — through + `SessionCancelHandle`, including straight after the call that started the + action (`test/openjd/sessions/test_refused_calls.py::TestCancelAction`). 2. Validate `run_subprocess(timeout=...)` and `run_subprocess(command=...)` arguments at the **wrapper** boundary so timeout=0/negative and empty command raise `ValueError` matching reference. Reference tests: @@ -555,12 +558,16 @@ Numbered for the report-driven workflow in `~/openjd-rs/AGENTS.md`. Strike throu `::test_run_subprocess_rejects_negative_timeout`, `::test_run_subprocess_rejects_empty_command`. Files: `openjd-sessions-for-python/src/openjd/sessions/_v1/_session.py` `Session.run_subprocess` (replace `if timeout else None` with explicit - `is None` check; add empty-command check at top). + `is None` check; add empty-command check at top). **Partly resolved** — + the wrapper passes `timeout=0` through and the binding raises the crate's + `SessionError` for a zero timeout and a blank command (not `ValueError`). -3. In Rust, make negative `timeout` reject cleanly instead of panicking: +3. ~~In Rust, make negative `timeout` reject cleanly instead of panicking: replace `Duration::from_secs_f64(t).unwrap()` with a check that returns a `SessionError`. File: `rust-bindings/src/sessions/session.rs` line 425 - (`run_subprocess`'s `let duration = timeout.map(std::time::Duration::from_secs_f64);`). + (`run_subprocess`'s `let duration = timeout.map(std::time::Duration::from_secs_f64);`).~~ + **Resolved** — a negative, NaN or infinite `timeout` or `time_limit` raises + `ValueError`. 4. Fix `exit_environment` default `keep_session_running=False` to match reference. Reference test: diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index bcc5809b..24e6a441 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -160,8 +160,9 @@ fn failed_action_status(message: &str) -> ActionStatus { } } -/// Convert a timeout or time limit given in seconds. Zero passes through, so -/// the crate gives its own refusal for it. A negative, NaN or infinite value +/// Convert a timeout or time limit given in seconds. Zero passes through to +/// the crate, which refuses a zero `run_subprocess` timeout and reads a zero +/// cancel time limit as "terminate now". A negative, NaN or infinite value /// has no `Duration`, so it raises `ValueError` here rather than panicking. fn seconds_to_duration(name: &str, seconds: f64) -> PyResult { std::time::Duration::try_from_secs_f64(seconds).map_err(|_| { @@ -301,7 +302,6 @@ fn run_action( // would leave the Python poller waiting forever. Force a // terminal FAILED status and drop out of RUNNING, mirroring the // runtime-build-failure branch above. - drop(guard); { let mut snap = lock_recover(&snapshot); snap.state = SessionState::Ready; @@ -310,6 +310,7 @@ fn run_action( panic_detail(payload.as_ref()) ))); } + drop(guard); tell(call, true, None); } } @@ -359,6 +360,11 @@ fn panic_detail(payload: &(dyn std::any::Any + Send)) -> String { #[pyclass(module = "openjd.sessions._v1", name = "Session")] pub(crate) struct PySession { /// The Rust session. None while a background thread has taken ownership. + /// + /// Lock it only with the GIL released (`Python::detach`). `cleanup` and + /// `cancel_action` call into the crate while holding it, the crate logs, + /// and `pyo3_log` takes the GIL for each record, so a thread that held + /// the GIL while waiting for this lock would deadlock with them. session: Arc>>, /// Snapshot updated after each state change, readable without blocking on the session. snapshot: Arc>, @@ -409,15 +415,11 @@ impl PySession { + Send + 'static, { - let session = { - let mut guard = lock_recover(&self.session); - match guard.take() { - Some(session) => session, - None => { - drop(guard); - return Err(self.busy_error(expected)); - } - } + // The session mutex is only ever locked with the GIL released; see + // `PySession::session`. + let session_arc = self.session.clone(); + let Some(session) = py.detach(move || lock_recover(&session_arc).take()) else { + return Err(self.busy_error(expected)); }; let (tx, rx) = std::sync::mpsc::sync_channel(1); *lock_recover(&self.pending) = Some(PendingCall { @@ -444,7 +446,8 @@ impl PySession { if let Err(e) = spawned { *lock_recover(&self.pending) = None; if let Some(session) = lock_recover(&session_slot).take() { - *lock_recover(&self.session) = Some(session); + let session_arc = self.session.clone(); + py.detach(move || *lock_recover(&session_arc) = Some(session)); } return Err(session_err_to_py(SessionError::Runtime(format!( "failed to start the action thread: {e}" @@ -638,19 +641,23 @@ impl PySession { /// that is not absolute. fn extend_path_mapping_rules( &self, + py: Python<'_>, additional: Vec, ) -> PyResult<()> { - let mut guard = lock_recover(&self.session); - let session = guard.as_mut().ok_or_else(|| { - pyo3::exceptions::PyRuntimeError::new_err( - "cannot extend path mapping rules while an action is running", - ) - })?; let rules: Vec = additional.into_iter().map(|p| p.inner).collect(); - session - .extend_path_mapping_rules(rules) - .map_err(session_err_to_py) + let session_arc = self.session.clone(); + let result = py.detach(move || { + lock_recover(&session_arc) + .as_mut() + .map(|session| session.extend_path_mapping_rules(rules)) + }); + match result { + Some(r) => r.map_err(session_err_to_py), + None => Err(pyo3::exceptions::PyRuntimeError::new_err( + "cannot extend path mapping rules while an action is running", + )), + } } // ── Non-blocking action methods ── @@ -824,15 +831,20 @@ impl PySession { /// Raises `SessionError` if the session isn't `READY_ENDING`. The /// crate also refuses after its session-wide cancel token is canceled; /// this binding doesn't expose one, so that can't happen here. - fn clear_ending_only(&self) -> PyResult<()> { - let mut guard = lock_recover(&self.session); - let Some(session) = guard.as_mut() else { - drop(guard); - return Err(self.busy_error(&[SessionState::ReadyEnding])); - }; - session.clear_ending_only().map_err(session_err_to_py)?; - Self::update_snapshot(session, &self.snapshot); - Ok(()) + fn clear_ending_only(&self, py: Python<'_>) -> PyResult<()> { + let session_arc = self.session.clone(); + let snapshot = self.snapshot.clone(); + let result = py.detach(move || { + let mut guard = lock_recover(&session_arc); + let session = guard.as_mut()?; + let r = session.clear_ending_only(); + Self::update_snapshot(session, &snapshot); + Some(r) + }); + match result { + Some(r) => r.map_err(session_err_to_py), + None => Err(self.busy_error(&[SessionState::ReadyEnding])), + } } /// Cancel the running action. @@ -858,7 +870,12 @@ impl PySession { // action's cancel token, so a cancel straight after the call that // started it can find no token yet. Keep trying until the token is // there or the action thread has given the session back; the gap is - // well under a millisecond. The GIL is released meanwhile. + // well under a millisecond. Give up after `CANCEL_WAIT`, which is + // only reached when the action thread itself is waiting on this + // call (a logging handler that cancels, for example). The GIL is + // released meanwhile. + const CANCEL_WAIT: std::time::Duration = std::time::Duration::from_secs(2); + let deadline = std::time::Instant::now() + CANCEL_WAIT; py.detach(move || loop { { let mut guard = lock_recover(&session_arc); @@ -885,6 +902,11 @@ impl PySession { return Ok(()); } } + if std::time::Instant::now() >= deadline { + return Err(session_err_to_py(SessionError::Runtime( + "Cannot cancel: the action has not started yet".into(), + ))); + } std::thread::sleep(std::time::Duration::from_micros(100)); }) } diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index 648d6a12..f7599eb7 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -123,6 +123,7 @@ class Session: *, environment: Environment, identifier: Optional[str] = None, + resolved_symtab: Optional[SerializedSymbolTable] = None, os_env_vars: Optional[dict[str, str]] = None, ) -> str: ... @@ -130,6 +131,8 @@ class Session: self, *, identifier: str, + resolved_symtab: Optional[SerializedSymbolTable] = None, + keep_session_running: bool = True, os_env_vars: Optional[dict[str, str]] = None, ) -> None: ... @@ -138,7 +141,9 @@ class Session: self, *, step_script: StepScript, + step_name: Optional[str] = None, task_parameter_values: Optional[dict] = None, + resolved_symtab: Optional[SerializedSymbolTable] = None, os_env_vars: Optional[dict[str, str]] = None, ) -> None: ... @@ -157,9 +162,8 @@ class Session: # Control def cancel_action( self, - *, time_limit: Optional[float] = None, - mark_action_failed: bool = False, + mark_action_failed: Optional[bool] = None, ) -> None: ... def clear_ending_only(self) -> None: ... @@ -216,7 +220,9 @@ infinite raises `ValueError`. Once the crate accepts a call, the method returns when its action is `RUNNING` (or, for an environment with no script, has ended), and `action_status` describes that action. A `cancel_action` made as soon as -it returns reaches the action. Every error after acceptance, such as a +it returns reaches the action, unless the action has already ended, as +one that fails before its subprocess starts can: then it raises the +crate's `InvalidState`. A zero `time_limit` terminates at once. Every error after acceptance, such as a parameter value that can't be coerced, is reported on `action_status` and doesn't raise, even when it arrives before the call returns. The crate documents those as failing the action rather than refusing the @@ -238,7 +244,7 @@ with the table the environment was entered with (openjd-rs#536). White space around a number in an INT or FLOAT job or task parameter value is ignored, so `" 7 "` is `7` and `" 1.50 "` is `1.50` (openjd-rs#530). On Windows, `cleanup` retries removing a session directory that is -still in use, for up to about 31 s, and holds the GIL meanwhile +still in use, for up to about 31 s, with the GIL released (openjd-rs#485). ### SessionUser (stays in Python, passed to Rust) diff --git a/test/openjd/sessions/test_refused_calls.py b/test/openjd/sessions/test_refused_calls.py index dd0760bb..2f57296d 100644 --- a/test/openjd/sessions/test_refused_calls.py +++ b/test/openjd/sessions/test_refused_calls.py @@ -423,3 +423,91 @@ def test_an_environment_with_no_script_has_ended_on_return(self, session: Sessio assert session.state == SessionState.READY session.exit_environment(identifier=identifier) assert session.environments_entered == [] + + +_CONCURRENT_CLEANUP = r""" +import logging, os, sys, threading +logging.basicConfig(level=logging.INFO, stream=open(os.devnull, "w")) +from openjd import _openjd_rs +from openjd._openjd_rs import Session +SessionError = getattr(_openjd_rs, "SessionError") +for i in range(50): + s = Session(session_id=f"s{i}", job_parameter_values={}, session_root_directory=sys.argv[1]) + stop = threading.Event() + def hammer(): + while not stop.is_set(): + try: + s.clear_ending_only() + except SessionError: + pass + t = threading.Thread(target=hammer, daemon=True) + t.start() + s.cleanup() + stop.set() + t.join() +print("done") +""" + + +def test_cleanup_with_a_concurrent_call_and_logging_on(tmp_path: Path) -> None: + """``cleanup`` holds the session lock while the crate logs, and each log + record needs the GIL. A concurrent call that waited for the lock while + holding the GIL deadlocked. Every lock of the session is now taken with + the GIL released. This runs in a fresh interpreter so the INFO level is + in force when the binding's logger first looks it up.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-c", _CONCURRENT_CLEANUP, str(tmp_path)], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "done" + + +_CANCEL_ON_BANNER = r""" +import logging, sys, time +from openjd import _openjd_rs +from openjd._openjd_rs import Session, SessionState, deserialize_step +SessionError = getattr(_openjd_rs, "SessionError") +results = [] +class CancelOnBanner(logging.Handler): + def emit(self, record): + if "Running Task" in record.getMessage() and not results: + try: + session.cancel_action() + results.append("canceled") + except SessionError as e: + results.append(str(e)) +logging.getLogger().addHandler(CancelOnBanner()) +logging.getLogger().setLevel(logging.INFO) +session = Session(session_id="s", job_parameter_values={}, session_root_directory=sys.argv[1]) +script = deserialize_step( + {"name": "S", "script": {"actions": {"onRun": {"command": sys.executable, "args": ["-c", ""]}}}} +).script +session.run_task(step_script=script) +while session.state == SessionState.RUNNING: + time.sleep(0.02) +session.cleanup() +print(results) +""" + + +def test_cancel_from_the_action_thread_before_it_starts(tmp_path: Path) -> None: + """The crate logs the "Running Task" banner on the action thread between + reporting the action ``Running`` and installing its cancel token. A + logging handler that cancels there can't wait for the token, so after a + bounded wait it raises instead of hanging. Runs in a fresh interpreter, + as above.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-c", _CANCEL_ON_BANNER, str(tmp_path)], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "['Cannot cancel: the action has not started yet']" From 16b50ac198c670b80498e014c944b2f9c7d8bd9d Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:04:20 +0000 Subject: [PATCH 18/21] fix: Wait out the cancel-token gap from any thread but the action's own A third review found that a cancel straight after an action started could still be lost under load. The crate reports an action Running, then logs its banner and installs its cancel token; logging needs the GIL and runs every Python logging handler, so the gap is as long as the action thread waits for the GIL, seconds when other Python threads are busy. The previous commit gave up after 2 s everywhere. The binding now records which thread runs the current action, and only a cancel from that thread (a logging handler) gives up after 2 s; from any other thread it waits, with the GIL released, until the token is installed or the session is back, which the action thread does without help. Pinned in a fresh interpreter with a logging handler that takes 2.5 s. Also from the review: - If an action thread ever ended without telling its caller the outcome, the caller would have waited for ever. A guard held by the thread now drops the call's sender, only if the slot still holds that call, so the caller wakes up. - Every refusal test now checks state, action_status and environments_entered, including the busy, ValueError and clear_ending_only cases. The fresh-interpreter tests' timeouts are under pytest's 30 s limit. - The sessions spec's constructor block matches the binding, it documents the binding's own "Cannot cancel: the action has not started yet" and WindowsSessionUser's RuntimeError, and its new paragraphs are rewrapped. The quality report lists the WindowsSessionUser change. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- ...ions-bindings-quality-evaluation-report.md | 1 + rust-bindings/src/sessions/session.rs | 83 +++++++++++++--- specs/python-sessions-interface.md | 67 +++++++------ src/openjd/_openjd_rs.pyi | 5 +- test/openjd/sessions/test_refused_calls.py | 94 ++++++++++++++----- 5 files changed, 180 insertions(+), 70 deletions(-) diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index 6bc7892f..b1b6b43b 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -372,6 +372,7 @@ counted as test-suite failures. | `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `SessionError("Session must be in READY state, current: RUNNING")` (the crate's message; a `RuntimeError`) | ⚠ message change | | `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | `SessionError("Session must be in READY state, current: READY_ENDING")`, a `RuntimeError`; nothing changes | ⚠ message change (the crate's) | | `Session.clear_ending_only()` | not in reference | takes a `READY_ENDING` session back to `READY`, as the crate's `clear_ending_only` does | ⚠ binding-only API | +| `WindowsSessionUser(...)` when the process user can't be looked up | `get_process_user()` raises | raises `RuntimeError("Could not determine the process user: ...")` (before, treated as "not the process user") | ⚠ class change | | `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change (the crate's) | | `cancel_action` while running | succeeds | succeeds, including straight after the call that started the action | ✓ resolved | | `Session(session_root_directory=…)` non-existent dir | `RuntimeError("Ensure that the root directory ...")` | likely Rust I/O error → `SessionError(...)` (untested) | ⚠ untested | diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index 24e6a441..668fb1a6 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -198,6 +198,8 @@ enum CallOutcome { /// The call an action thread is running, until the Python thread that made /// it has been told the outcome. struct PendingCall { + /// Tells this call apart from the next one in the same slot. + id: u64, tx: std::sync::mpsc::SyncSender, /// Set by the session callback: the crate reported an action for this /// call, so it accepted it. @@ -206,6 +208,39 @@ struct PendingCall { type PendingSlot = Arc>>; +static NEXT_CALL_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// The thread running a session's current action, so `cancel_action` can +/// tell whether it is called from that thread. +type ActionThread = Arc>>; + +/// Held by an action thread for its whole life. If the thread ends without +/// having told its caller the outcome (it can't today, but a panic outside +/// `catch_unwind` would), dropping the call's sender wakes the caller instead +/// of leaving it waiting for ever. Also clears the thread's entry in +/// `ActionThread`. Each slot is cleared only while it still names this call +/// or thread, not a later one. +struct ActionThreadGuard { + pending: PendingSlot, + call_id: u64, + action_thread: ActionThread, +} + +impl Drop for ActionThreadGuard { + fn drop(&mut self) { + { + let mut slot = lock_recover(&self.pending); + if slot.as_ref().is_some_and(|c| c.id == self.call_id) { + slot.take(); + } + } + let mut thread = lock_recover(&self.action_thread); + if *thread == Some(std::thread::current().id()) { + *thread = None; + } + } +} + /// Body of a spawned action thread. /// /// Builds a lightweight current-thread Tokio runtime, runs the action @@ -376,6 +411,8 @@ pub(crate) struct PySession { /// The call whose caller is waiting to learn whether the crate accepted /// it. Shared with the session callback, which marks it accepted. pending: PendingSlot, + /// The thread running the current action, if any. + action_thread: ActionThread, } impl PySession { @@ -422,7 +459,9 @@ impl PySession { return Err(self.busy_error(expected)); }; let (tx, rx) = std::sync::mpsc::sync_channel(1); + let call_id = NEXT_CALL_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); *lock_recover(&self.pending) = Some(PendingCall { + id: call_id, tx, accepted: false, }); @@ -435,9 +474,16 @@ impl PySession { let session_arc = self.session.clone(); let snapshot = self.snapshot.clone(); let pending = self.pending.clone(); + let action_thread = self.action_thread.clone(); let spawned = std::thread::Builder::new() .name("openjd-session-action".into()) .spawn(move || { + *lock_recover(&action_thread) = Some(std::thread::current().id()); + let _guard = ActionThreadGuard { + pending: pending.clone(), + call_id, + action_thread, + }; let Some(session) = lock_recover(&for_thread).take() else { return; }; @@ -459,9 +505,10 @@ impl PySession { accepted: false, error: Some(e), }) => Err(session_err_to_py(e)), - // Started, an accepted call that returned (its status says how), - // or an action thread that ended without saying (it panicked - // outside `catch_unwind`; the snapshot reports the failure). + // Started, or an accepted call that returned (its status says + // how). `Err` means the action thread ended without saying, which + // `ActionThreadGuard` turns into a wake-up; nothing does that + // today. _ => Ok(()), } } @@ -586,6 +633,7 @@ impl PySession { snapshot, cancel_handle, pending, + action_thread: Arc::new(Mutex::new(None)), }) } @@ -851,7 +899,10 @@ impl PySession { /// /// Raises `SessionError` with the crate's message if no action is /// running, and `ValueError` if `time_limit` is negative, NaN or - /// infinite. + /// infinite. Called from the action's own thread (from a logging + /// handler) before the action has started its subprocess, it waits up + /// to 2 s and then raises `SessionError("Cannot cancel: the action has + /// not started yet")`. #[pyo3(signature = (time_limit=None, mark_action_failed=None))] fn cancel_action( &self, @@ -866,16 +917,20 @@ impl PySession { let session_arc = self.session.clone(); let snapshot = self.snapshot.clone(); let handle = self.cancel_handle.clone(); - // The crate reports an action `Running` just before it installs the - // action's cancel token, so a cancel straight after the call that - // started it can find no token yet. Keep trying until the token is - // there or the action thread has given the session back; the gap is - // well under a millisecond. Give up after `CANCEL_WAIT`, which is - // only reached when the action thread itself is waiting on this - // call (a logging handler that cancels, for example). The GIL is - // released meanwhile. + // The crate reports an action `Running` just before it logs the + // action's banner and installs its cancel token, so a cancel straight + // after the call that started it can find no token yet. Logging needs + // the GIL, so the gap is as long as the action thread waits for it, + // which under load can be seconds. Keep trying, with the GIL + // released, until the token is there or the action thread has given + // the session back; both happen without help from this thread. On + // the action thread itself (a logging handler that cancels, say), + // neither can happen while this call waits, so give up after + // `CANCEL_WAIT` there. const CANCEL_WAIT: std::time::Duration = std::time::Duration::from_secs(2); - let deadline = std::time::Instant::now() + CANCEL_WAIT; + let on_action_thread = + *lock_recover(&self.action_thread) == Some(std::thread::current().id()); + let deadline = on_action_thread.then(|| std::time::Instant::now() + CANCEL_WAIT); py.detach(move || loop { { let mut guard = lock_recover(&session_arc); @@ -902,7 +957,7 @@ impl PySession { return Ok(()); } } - if std::time::Instant::now() >= deadline { + if deadline.is_some_and(|d| std::time::Instant::now() >= d) { return Err(session_err_to_py(SessionError::Runtime( "Cannot cancel: the action has not started yet".into(), ))); diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index f7599eb7..bf74759f 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -100,13 +100,11 @@ class Session: session_id: str, job_parameter_values: dict[str, ParameterValue], path_mapping_rules: Optional[list[PathMappingRule]] = None, - callback: Optional[Callable] = None, - os_env_vars: Optional[dict[str, str]] = None, retain_working_dir: bool = False, + os_env_vars: Optional[dict[str, str]] = None, session_root_directory: Optional[Path] = None, user: Optional[SessionUser] = None, - job_template: Optional[JobTemplate] = None, - environment_templates: Optional[list[EnvironmentTemplate]] = None, + profile: Optional[ModelProfile] = None, caller_limits: Optional[CallerLimits] = None, ): ... @@ -116,6 +114,9 @@ class Session: working_directory: Path files_directory: Path action_status: Optional[ActionStatus] + environments_entered: list[str] + + def extend_path_mapping_rules(self, additional: list[PathMappingRule]) -> None: ... # Environment lifecycle (non-blocking — returns once the action is running) def enter_environment( @@ -208,13 +209,13 @@ the state, `action_status` (still the previous action's) and `environments_entered` stay as they were. The crate refuses `run_task`, `enter_environment` and `run_subprocess` unless the session is `READY`, `exit_environment` unless it is `READY` or `READY_ENDING`, -`cancel_action` unless an action is running, an unknown, out-of-order -or duplicate environment identifier, a second environment with wrap -hooks, and a blank `command` or zero `timeout`. A call made while -another action runs gets the crate's `InvalidState` message, such as -`Session must be in READY state, current: RUNNING`; before, it was -`RuntimeError("An action is already running")`, and `SessionError` is -a `RuntimeError`. A `timeout` or `time_limit` that is negative, NaN or +`cancel_action` unless an action is running, an unknown, out-of-order or +duplicate environment identifier, a second environment with wrap hooks, +and a blank `command` or zero `timeout`. A call made while another +action runs gets the crate's `InvalidState` message, such as `Session +must be in READY state, current: RUNNING`; before, it was +`RuntimeError("An action is already running")`, and `SessionError` is a +`RuntimeError`. A `timeout` or `time_limit` that is negative, NaN or infinite raises `ValueError`. Once the crate accepts a call, the method returns when its action is @@ -222,30 +223,34 @@ Once the crate accepts a call, the method returns when its action is `action_status` describes that action. A `cancel_action` made as soon as it returns reaches the action, unless the action has already ended, as one that fails before its subprocess starts can: then it raises the -crate's `InvalidState`. A zero `time_limit` terminates at once. Every error after acceptance, such as a -parameter value that can't be coerced, is reported on `action_status` -and doesn't raise, even when it arrives before the call returns. The -crate documents those as failing the action rather than refusing the -call, and the pure-Python reference reports them the same way. +crate's `InvalidState`. A zero `time_limit` terminates at once. Called +from the action's own thread (a logging handler) before the action has +started, `cancel_action` waits up to 2 s and then raises +`SessionError("Cannot cancel: the action has not started yet")`, the +binding's own message. Every error after acceptance, such as a parameter +value that can't be coerced, is reported on `action_status` and doesn't +raise, even when it arrives before the call returns. The crate documents +those as failing the action rather than refusing the call, and the +pure-Python reference reports them the same way. An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session ending-only (`READY_ENDING`): `exit_environment` and `cleanup` still work, and a successful `exit_environment(keep_session_running=True)` leaves it `READY_ENDING` (openjd-rs#524). `clear_ending_only()` takes it back to `READY`, keeping the environments entered and the last action's -status; it raises `SessionError` unless the session is `READY_ENDING`. When `run_task`, `enter_environment` or -`exit_environment` fails before its subprocess starts, for example -because a parameter value can't be coerced, the action is `FAILED` with -the error's message, as in the pure-Python reference (openjd-rs#532). -An environment whose enter fails this way still counts as entered, and -one whose exit fails this way counts as exited. When `exit_environment` -can't build its symbol table from `resolved_symtab`, it runs `onExit` -with the table the environment was entered with (openjd-rs#536). White -space around a number in an INT or FLOAT job or task parameter value is -ignored, so `" 7 "` is `7` and `" 1.50 "` is `1.50` (openjd-rs#530). -On Windows, `cleanup` retries removing a session directory that is -still in use, for up to about 31 s, with the GIL released -(openjd-rs#485). +status; it raises `SessionError` unless the session is `READY_ENDING`. +When `run_task`, `enter_environment` or `exit_environment` fails before +its subprocess starts, for example because a parameter value can't be +coerced, the action is `FAILED` with the error's message, as in the +pure-Python reference (openjd-rs#532). An environment whose enter fails +this way still counts as entered, and one whose exit fails this way +counts as exited. When `exit_environment` can't build its symbol table +from `resolved_symtab`, it runs `onExit` with the table the environment +was entered with (openjd-rs#536). White space around a number in an INT +or FLOAT job or task parameter value is ignored, so `" 7 "` is `7` and +`" 1.50 "` is `1.50` (openjd-rs#530). On Windows, `cleanup` retries +removing a session directory that is still in use, for up to about 31 s, +with the GIL released (openjd-rs#485). ### SessionUser (stays in Python, passed to Rust) @@ -264,6 +269,10 @@ class WindowsSessionUser: password: Optional[str] ``` +`WindowsSessionUser` raises `RuntimeError("Could not determine the +process user: ...")` if it can't look up the process user, as the +pure-Python reference's `get_process_user()` raises. + ### Callback interface The Python callback `Callable[[str, ActionStatus], None]` is invoked from diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index 6d722dfa..ccb72e43 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -2303,7 +2303,10 @@ class Session: Raises `SessionError` with the crate's message if no action is running, and `ValueError` if `time_limit` is negative, NaN or - infinite. + infinite. Called from the action's own thread (from a logging + handler) before the action has started its subprocess, it waits up + to 2 s and then raises `SessionError("Cannot cancel: the action has + not started yet")`. """ def cleanup(self) -> None: diff --git a/test/openjd/sessions/test_refused_calls.py b/test/openjd/sessions/test_refused_calls.py index 2f57296d..a5894510 100644 --- a/test/openjd/sessions/test_refused_calls.py +++ b/test/openjd/sessions/test_refused_calls.py @@ -87,12 +87,17 @@ def session(tmp_path: Path): s.cleanup() -def _assert_refused(session: Session, call: Callable[[], object], message: str) -> None: - """``call`` raises ``SessionError(message)`` and changes nothing.""" +def _assert_refused( + session: Session, + call: Callable[[], object], + message: str, + error: type[Exception] = SessionError, +) -> None: + """``call`` raises ``error(message)`` and changes nothing.""" state = session.state status = _status(session) entered = session.environments_entered - with pytest.raises(SessionError) as exc_info: + with pytest.raises(error) as exc_info: call() assert str(exc_info.value) == message assert session.state == state @@ -197,10 +202,10 @@ def test_refused_while_an_action_runs(self, session: Session) -> None: session.run_task(step_script=_step_script("import time; time.sleep(30)")) try: assert session.state == SessionState.RUNNING - with pytest.raises(SessionError) as exc_info: - session.clear_ending_only() - assert str(exc_info.value) == ( - "Session must be in READY_ENDING state, current: RUNNING" + _assert_refused( + session, + session.clear_ending_only, + "Session must be in READY_ENDING state, current: RUNNING", ) finally: session.cancel_action() @@ -244,13 +249,8 @@ def test_a_second_action( session.run_task(step_script=_step_script("import time; time.sleep(30)")) try: assert session.state == SessionState.RUNNING - running = _status(session) - with pytest.raises(SessionError) as exc_info: - call(session) - assert isinstance(exc_info.value, RuntimeError) - assert str(exc_info.value) == message - assert session.state == SessionState.RUNNING - assert _status(session) == running + assert issubclass(SessionError, RuntimeError) + _assert_refused(session, lambda: call(session), message) finally: session.cancel_action() _settle(session) @@ -328,13 +328,12 @@ def test_run_subprocess_timeout_with_no_duration( """A timeout with no ``Duration`` raises ``ValueError`` before the crate sees it. Before, the binding's conversion panicked (``PanicException``, which ``except Exception`` doesn't catch).""" - with pytest.raises(ValueError) as exc_info: - session.run_subprocess(command=sys.executable, timeout=timeout) - assert str(exc_info.value) == ( - f"timeout must be a finite, non-negative number of seconds, got {shown}" + _assert_refused( + session, + lambda: session.run_subprocess(command=sys.executable, timeout=timeout), + f"timeout must be a finite, non-negative number of seconds, got {shown}", + ValueError, ) - assert session.state == SessionState.READY - assert session.action_status is None _SLEEP = "import time; time.sleep(30)" @@ -350,10 +349,11 @@ def test_with_no_action_running(self, session: Session) -> None: _assert_refused(session, session.cancel_action, message) def test_a_time_limit_with_no_duration(self, session: Session) -> None: - with pytest.raises(ValueError) as exc_info: - session.cancel_action(time_limit=-1.0) - assert str(exc_info.value) == ( - "time_limit must be a finite, non-negative number of seconds, got -1" + _assert_refused( + session, + lambda: session.cancel_action(time_limit=-1.0), + "time_limit must be a finite, non-negative number of seconds, got -1", + ValueError, ) @pytest.mark.parametrize( @@ -461,7 +461,7 @@ def test_cleanup_with_a_concurrent_call_and_logging_on(tmp_path: Path) -> None: [sys.executable, "-c", _CONCURRENT_CLEANUP, str(tmp_path)], capture_output=True, text=True, - timeout=120, + timeout=25, ) assert result.returncode == 0, result.stderr assert result.stdout.strip() == "done" @@ -507,7 +507,49 @@ def test_cancel_from_the_action_thread_before_it_starts(tmp_path: Path) -> None: [sys.executable, "-c", _CANCEL_ON_BANNER, str(tmp_path)], capture_output=True, text=True, - timeout=120, + timeout=25, ) assert result.returncode == 0, result.stderr assert result.stdout.strip() == "['Cannot cancel: the action has not started yet']" + + +_CANCEL_WHILE_THE_ACTION_THREAD_WAITS = r""" +import logging, sys, time +from openjd._openjd_rs import ActionState, Session, SessionState, deserialize_step +class SlowOnBanner(logging.Handler): + def emit(self, record): + if "Running Task" in record.getMessage(): + time.sleep(2.5) +logging.getLogger().addHandler(SlowOnBanner()) +logging.getLogger().setLevel(logging.INFO) +session = Session(session_id="s", job_parameter_values={}, session_root_directory=sys.argv[1]) +script = deserialize_step( + {"name": "S", "script": {"actions": {"onRun": { + "command": sys.executable, "args": ["-c", "import time; time.sleep(30)"]}}}} +).script +session.run_task(step_script=script) +session.cancel_action() +while session.state in (SessionState.RUNNING, SessionState.CANCELING): + time.sleep(0.02) +print(session.action_status.state.name) +session.cleanup() +""" + + +def test_cancel_while_the_action_thread_is_slow_to_start(tmp_path: Path) -> None: + """The gap between the crate reporting ``Running`` and installing the + cancel token includes logging the action's banner, which needs the GIL + and runs every Python logging handler. Here a handler takes 2.5 s. A + cancel from another thread waits the gap out rather than giving up; with + a 2 s limit it used to raise "Cannot cancel: the action has not started + yet", and the task ran to the end.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-c", _CANCEL_WHILE_THE_ACTION_THREAD_WAITS, str(tmp_path)], + capture_output=True, + text=True, + timeout=25, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "CANCELED" From c69d79e469e765178e17c9b4777b5f8f2d7f5543 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:20:09 +0000 Subject: [PATCH 19/21] fix: Refuse a second cancel, and bound every cancel-token wait A fourth review found: - A second cancel_action while the session was CANCELING went through the cancel handle and returned, ignoring its time limit. The crate's Session::cancel_action refuses it with InvalidState; the binding now does the same. - The unbounded wait for the cancel token could hang for ever when the caller held a lock that a logging handler on the action thread needs, since the action thread logs its banner before installing the token. The wait is now bounded at 20 s from other threads, then raises "Cannot cancel: the action has not started yet". On the action thread itself it raises at once, since the token can't appear while that thread waits. - dispatch turned a lost wake-up from the action thread into success. It now raises SessionError "the action thread ended unexpectedly". Nothing produces that today. The sessions spec now says not to call an action method or cancel_action while holding a lock that a logging handler takes on another thread, since each waits for the action thread, which logs. Fix stale comments about the runtime-build failure, and keep the tests' waits under pytest's 30 s limit. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- rust-bindings/src/sessions/session.rs | 81 ++++++++++++++-------- specs/python-sessions-interface.md | 28 +++++--- src/openjd/_openjd_rs.pyi | 12 ++-- test/openjd/sessions/test_refused_calls.py | 57 +++++++++++++-- 4 files changed, 129 insertions(+), 49 deletions(-) diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index 668fb1a6..1e9d681d 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -145,9 +145,9 @@ fn lock_recover(m: &Mutex) -> std::sync::MutexGuard<'_, T> { /// Build a terminal FAILED `ActionStatus` carrying `message`. /// -/// Used when an action thread can't even start (e.g. the Tokio runtime -/// fails to build) so the Python poller observes a terminal state and -/// stops waiting on RUNNING rather than spinning forever. +/// Used when an accepted action panics, or returns an error the crate +/// didn't record, so the Python poller observes a terminal state and stops +/// waiting on RUNNING rather than spinning forever. fn failed_action_status(message: &str) -> ActionStatus { ActionStatus { state: openjd_sessions::action::ActionState::Failed, @@ -254,11 +254,11 @@ impl Drop for ActionThreadGuard { /// poison shared state silently and tear down the interpreter's view /// of the session. /// -/// A runtime-build failure is surfaced as a terminal FAILED -/// `ActionStatus` so the Python-side poller terminates cleanly. A panic -/// inside the action is likewise converted into a terminal FAILED -/// `ActionStatus` (rather than copying the possibly-still-Running real -/// session state), so the poller never hangs on a panicked action. +/// A runtime-build failure is a refusal: the session goes back unchanged +/// and the caller raises. A panic inside the action is converted into a +/// terminal FAILED `ActionStatus` (rather than copying the +/// possibly-still-Running real session state), so the poller never hangs +/// on a panicked action. /// /// If the calling Python thread is still waiting (the action never /// reported `Running`), it is told the outcome after the session is back in @@ -335,8 +335,7 @@ fn run_action( // usable for future calls, but the real `session.state()` may // still read Running with no terminal `ActionStatus` — which // would leave the Python poller waiting forever. Force a - // terminal FAILED status and drop out of RUNNING, mirroring the - // runtime-build-failure branch above. + // terminal FAILED status and drop out of RUNNING. { let mut snap = lock_recover(&snapshot); snap.state = SessionState::Ready; @@ -505,11 +504,15 @@ impl PySession { accepted: false, error: Some(e), }) => Err(session_err_to_py(e)), + // `Err`: the action thread ended without saying, which + // `ActionThreadGuard` turns into a wake-up. Nothing does that + // today; if it did, the session may not be back in its slot. + Err(_) => Err(session_err_to_py(SessionError::Runtime( + "the action thread ended unexpectedly".into(), + ))), // Started, or an accepted call that returned (its status says - // how). `Err` means the action thread ended without saying, which - // `ActionThreadGuard` turns into a wake-up; nothing does that - // today. - _ => Ok(()), + // how). + Ok(_) => Ok(()), } } } @@ -898,11 +901,13 @@ impl PySession { /// Cancel the running action. /// /// Raises `SessionError` with the crate's message if no action is - /// running, and `ValueError` if `time_limit` is negative, NaN or - /// infinite. Called from the action's own thread (from a logging - /// handler) before the action has started its subprocess, it waits up - /// to 2 s and then raises `SessionError("Cannot cancel: the action has - /// not started yet")`. + /// running or one is already being canceled, and `ValueError` if + /// `time_limit` is negative, NaN or infinite. Raises + /// `SessionError("Cannot cancel: the action has not started yet")` if + /// the action hasn't installed its cancel token: at once when called + /// from the action's own thread (a logging handler), and after 20 s + /// otherwise, which happens only when the action thread is blocked on a + /// lock the caller holds, such as a logging handler's. #[pyo3(signature = (time_limit=None, mark_action_failed=None))] fn cancel_action( &self, @@ -920,17 +925,23 @@ impl PySession { // The crate reports an action `Running` just before it logs the // action's banner and installs its cancel token, so a cancel straight // after the call that started it can find no token yet. Logging needs - // the GIL, so the gap is as long as the action thread waits for it, - // which under load can be seconds. Keep trying, with the GIL - // released, until the token is there or the action thread has given - // the session back; both happen without help from this thread. On - // the action thread itself (a logging handler that cancels, say), - // neither can happen while this call waits, so give up after - // `CANCEL_WAIT` there. - const CANCEL_WAIT: std::time::Duration = std::time::Duration::from_secs(2); + // the GIL and runs every Python handler, so the gap is as long as the + // action thread waits for those, which under load can be seconds. + // Keep trying, with the GIL released, until the token is there or the + // action thread has given the session back. Give up after + // `CANCEL_WAIT`, which is reached only when the action thread is + // stuck on something this thread holds, such as a logging handler's + // lock. On the action thread itself (a logging handler that cancels) + // the token can't appear while this call runs, so don't wait at all. + const CANCEL_WAIT: std::time::Duration = std::time::Duration::from_secs(20); let on_action_thread = *lock_recover(&self.action_thread) == Some(std::thread::current().id()); - let deadline = on_action_thread.then(|| std::time::Instant::now() + CANCEL_WAIT); + let deadline = std::time::Instant::now() + + if on_action_thread { + std::time::Duration::ZERO + } else { + CANCEL_WAIT + }; py.detach(move || loop { { let mut guard = lock_recover(&session_arc); @@ -952,12 +963,24 @@ impl PySession { // write its final snapshot before the `Canceling` below, so // the snapshot can't be left reading `Canceling` after the // action has ended. + // A second cancel while one is in progress: the crate's + // `Session::cancel_action` refuses it, so do the same rather + // than deliver it through the handle. + { + let snap = lock_recover(&snapshot); + if snap.state == SessionState::Canceling { + return Err(session_err_to_py(SessionError::InvalidState { + expected: vec![SessionState::Running], + current: SessionState::Canceling, + })); + } + } if handle.cancel(duration, mark_failed) { lock_recover(&snapshot).state = SessionState::Canceling; return Ok(()); } } - if deadline.is_some_and(|d| std::time::Instant::now() >= d) { + if std::time::Instant::now() >= deadline { return Err(session_err_to_py(SessionError::Runtime( "Cannot cancel: the action has not started yet".into(), ))); diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index bf74759f..dff150f5 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -223,15 +223,25 @@ Once the crate accepts a call, the method returns when its action is `action_status` describes that action. A `cancel_action` made as soon as it returns reaches the action, unless the action has already ended, as one that fails before its subprocess starts can: then it raises the -crate's `InvalidState`. A zero `time_limit` terminates at once. Called -from the action's own thread (a logging handler) before the action has -started, `cancel_action` waits up to 2 s and then raises -`SessionError("Cannot cancel: the action has not started yet")`, the -binding's own message. Every error after acceptance, such as a parameter -value that can't be coerced, is reported on `action_status` and doesn't -raise, even when it arrives before the call returns. The crate documents -those as failing the action rather than refusing the call, and the -pure-Python reference reports them the same way. +crate's `InvalidState`. A zero `time_limit` terminates at once. A second +`cancel_action` while the session is `CANCELING` raises the crate's +`InvalidState`. If the action hasn't installed its cancel token yet, +`cancel_action` raises `SessionError("Cannot cancel: the action has not +started yet")`, the binding's own message: at once when called from the +action's own thread (a logging handler), and after 20 s otherwise. Every +error after acceptance, such as a parameter value that can't be coerced, +is reported on `action_status` and doesn't raise, even when it arrives +before the call returns. The crate documents those as failing the action +rather than refusing the call, and the pure-Python reference reports +them the same way. + +The crate runs each action on a background thread, and that thread logs +through Python's `logging` (each record needs the GIL and runs the +handlers). An action method waits for the crate to accept or refuse the +call, and `cancel_action` waits for the action's cancel token, so don't +call either while holding a lock that a logging handler takes on another +thread: the action thread would wait for the lock, and the call for the +action thread. An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session ending-only (`READY_ENDING`): `exit_environment` and `cleanup` still diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index ccb72e43..3322949e 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -2302,11 +2302,13 @@ class Session: Cancel the running action. Raises `SessionError` with the crate's message if no action is - running, and `ValueError` if `time_limit` is negative, NaN or - infinite. Called from the action's own thread (from a logging - handler) before the action has started its subprocess, it waits up - to 2 s and then raises `SessionError("Cannot cancel: the action has - not started yet")`. + running or one is already being canceled, and `ValueError` if + `time_limit` is negative, NaN or infinite. Raises + `SessionError("Cannot cancel: the action has not started yet")` if + the action hasn't installed its cancel token: at once when called + from the action's own thread (a logging handler), and after 20 s + otherwise, which happens only when the action thread is blocked on a + lock the caller holds, such as a logging handler's. """ def cleanup(self) -> None: diff --git a/test/openjd/sessions/test_refused_calls.py b/test/openjd/sessions/test_refused_calls.py index a5894510..5701ce18 100644 --- a/test/openjd/sessions/test_refused_calls.py +++ b/test/openjd/sessions/test_refused_calls.py @@ -64,11 +64,11 @@ def _environment(name: str = "E", *, extensions: Optional[list[str]] = None, **a def _settle(session: Session) -> None: - deadline = time.monotonic() + 30 + deadline = time.monotonic() + 10 busy = (SessionState.RUNNING, SessionState.CANCELING) while session.state in busy and time.monotonic() < deadline: time.sleep(0.02) - assert session.state not in busy, "action did not finish within 30s" + assert session.state not in busy, "action did not finish within 10s" def _status(session: Session) -> Optional[tuple]: @@ -348,6 +348,51 @@ def test_with_no_action_running(self, session: Session) -> None: _settle(session) _assert_refused(session, session.cancel_action, message) + def test_a_second_cancel_while_canceling(self, session: Session, tmp_path: Path) -> None: + """The crate refuses a cancel while one is in progress. Before, the + binding delivered it through the cancel handle and returned, ignoring + its time limit.""" + marker = tmp_path / "started" + ignore_notify = ( + "import signal, time\n" + "for name in ('SIGTERM', 'SIGBREAK'):\n" + " if hasattr(signal, name):\n" + " signal.signal(getattr(signal, name), signal.SIG_IGN)\n" + f"open({str(marker)!r}, 'w').close()\n" + "time.sleep(30)" + ) + script = deserialize_step( + { + "name": "S", + "script": { + "actions": { + "onRun": { + **_python(ignore_notify), + "cancelation": { + "mode": "NOTIFY_THEN_TERMINATE", + "notifyPeriodInSeconds": 3, + }, + } + } + }, + } + ).script + session.run_task(step_script=script) + deadline = time.monotonic() + 10 + while not marker.exists() and time.monotonic() < deadline: + time.sleep(0.02) + assert marker.exists() + session.cancel_action() + assert session.state == SessionState.CANCELING + _assert_refused( + session, + lambda: session.cancel_action(time_limit=0.0), + "Session must be in RUNNING state, current: CANCELING", + ) + _settle(session) + status = session.action_status + assert status is not None and status.state == ActionState.CANCELED + def test_a_time_limit_with_no_duration(self, session: Session) -> None: _assert_refused( session, @@ -382,7 +427,7 @@ def test_straight_after_the_call_that_started_the_action( began = time.monotonic() session.cancel_action() _settle(session) - assert time.monotonic() - began < 20 + assert time.monotonic() - began < 4 status = session.action_status assert status is not None and status.state == ActionState.CANCELED assert session.state == SessionState.READY_ENDING @@ -498,9 +543,9 @@ def emit(self, record): def test_cancel_from_the_action_thread_before_it_starts(tmp_path: Path) -> None: """The crate logs the "Running Task" banner on the action thread between reporting the action ``Running`` and installing its cancel token. A - logging handler that cancels there can't wait for the token, so after a - bounded wait it raises instead of hanging. Runs in a fresh interpreter, - as above.""" + logging handler that cancels there can't wait for the token, which can't + appear while it runs, so it raises at once instead of hanging. Runs in a + fresh interpreter, as above.""" import subprocess result = subprocess.run( From edcd19fbb62be2b165064d8ac580c16a5dd18fed Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:34:34 +0000 Subject: [PATCH 20/21] docs: Note the second-cancel change from the reference, and the logging-lock rule A review noted that refusing a second cancel_action while CANCELING, as the crate does, changes behavior for callers of the pure-Python Session, which accepts it and shortens the grace period to the new time limit. Say so in the cancel_action docstring, the sessions spec and the quality report. The crate can't shorten a cancel in progress. Put the rule against calling an action method while holding a lock that a logging handler takes in each action method's docstring, not only the spec, and say in the spec where the fix belongs (the crate should report acceptance and install the cancel token before logging). Say "typically" for when the 20 s cancel wait ends. Pin that a cancel from the action thread raises at once, and loosen the straight-after cancel test's bound to 8 s for slow CI runners. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- ...ions-bindings-quality-evaluation-report.md | 1 + rust-bindings/src/sessions/session.rs | 28 +++++++++++++++---- specs/python-sessions-interface.md | 25 ++++++++++------- src/openjd/_openjd_rs.pyi | 20 +++++++++++-- test/openjd/sessions/test_refused_calls.py | 6 ++-- 5 files changed, 60 insertions(+), 20 deletions(-) diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index b1b6b43b..7cbfa43c 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -371,6 +371,7 @@ counted as test-suite failures. |---|---|---|---| | `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `SessionError("Session must be in READY state, current: RUNNING")` (the crate's message; a `RuntimeError`) | ⚠ message change | | `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | `SessionError("Session must be in READY state, current: READY_ENDING")`, a `RuntimeError`; nothing changes | ⚠ message change (the crate's) | +| `cancel_action` while a cancel is in progress | accepted; shortens the grace period to the new `time_limit` | `SessionError("Session must be in RUNNING state, current: CANCELING")`, the crate's refusal; the crate can't shorten a cancel in progress | ⚠ behavior change | | `Session.clear_ending_only()` | not in reference | takes a `READY_ENDING` session back to `READY`, as the crate's `clear_ending_only` does | ⚠ binding-only API | | `WindowsSessionUser(...)` when the process user can't be looked up | `get_process_user()` raises | raises `RuntimeError("Could not determine the process user: ...")` (before, treated as "not the process user") | ⚠ class change | | `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change (the crate's) | diff --git a/rust-bindings/src/sessions/session.rs b/rust-bindings/src/sessions/session.rs index 1e9d681d..9002cfc7 100644 --- a/rust-bindings/src/sessions/session.rs +++ b/rust-bindings/src/sessions/session.rs @@ -730,6 +730,9 @@ impl PySession { /// wrap hook is entered. A failure after that, such as an environment /// variable that doesn't resolve, fails the action; the environment /// still counts as entered and must be exited. + /// + /// Waits for the action thread, which logs through Python's `logging`, + /// so don't call it while holding a lock that a logging handler takes. #[pyo3(signature = (*, environment, identifier=None, resolved_symtab=None, os_env_vars=None))] fn enter_environment( &self, @@ -776,6 +779,9 @@ impl PySession { /// `READY_ENDING`, if `identifier` isn't entered, or if it isn't the /// most recently entered environment. A failure after that fails the /// action, and the environment still counts as exited. + /// + /// Waits for the action thread, which logs through Python's `logging`, + /// so don't call it while holding a lock that a logging handler takes. #[pyo3(signature = (*, identifier, resolved_symtab=None, keep_session_running=true, os_env_vars=None))] fn exit_environment( &self, @@ -808,6 +814,9 @@ impl PySession { /// failure after that, such as a task parameter value that can't be /// coerced, fails the action. /// + /// Waits for the action thread, which logs through Python's `logging`, + /// so don't call it while holding a lock that a logging handler takes. + /// /// `step_name` is surfaced as `WrappedStep.Name` to a wrapping /// environment's `onWrapTaskRun` hook (RFC 0008). It defaults to an empty /// string when the caller does not supply one. @@ -846,6 +855,9 @@ impl PySession { /// Raises `SessionError` if the session isn't `READY`, if `command` is /// empty or blank, or if `timeout` is zero, and `ValueError` if /// `timeout` is negative, NaN or infinite. + /// + /// Waits for the action thread, which logs through Python's `logging`, + /// so don't call it while holding a lock that a logging handler takes. #[pyo3(signature = (*, command, args=None, timeout=None, os_env_vars=None, use_session_env_vars=true, log_banner_message=None))] #[allow(clippy::too_many_arguments)] // PyO3 method mirrors a kwarg-rich public signature fn run_subprocess( @@ -906,8 +918,12 @@ impl PySession { /// `SessionError("Cannot cancel: the action has not started yet")` if /// the action hasn't installed its cancel token: at once when called /// from the action's own thread (a logging handler), and after 20 s - /// otherwise, which happens only when the action thread is blocked on a - /// lock the caller holds, such as a logging handler's. + /// otherwise, typically because the action thread is blocked on a lock + /// the caller holds, such as a logging handler's. + /// + /// Unlike the pure-Python reference, a second cancel while one is in + /// progress raises; the crate doesn't shorten the first cancel's grace + /// period. #[pyo3(signature = (time_limit=None, mark_action_failed=None))] fn cancel_action( &self, @@ -959,10 +975,6 @@ impl PySession { // through the thread-safe cancel handle, which follows the // action's own cancelation method (including cross-user // helper delivery), exactly like Session::cancel_action. - // Holding the session lock means the action thread can't - // write its final snapshot before the `Canceling` below, so - // the snapshot can't be left reading `Canceling` after the - // action has ended. // A second cancel while one is in progress: the crate's // `Session::cancel_action` refuses it, so do the same rather // than deliver it through the handle. @@ -975,6 +987,10 @@ impl PySession { })); } } + // Holding the session lock means the action thread can't + // write its final snapshot before the `Canceling` below, so + // the snapshot can't be left reading `Canceling` after the + // action has ended. if handle.cancel(duration, mark_failed) { lock_recover(&snapshot).state = SessionState::Canceling; return Ok(()); diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index dff150f5..361036dd 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -225,15 +225,18 @@ it returns reaches the action, unless the action has already ended, as one that fails before its subprocess starts can: then it raises the crate's `InvalidState`. A zero `time_limit` terminates at once. A second `cancel_action` while the session is `CANCELING` raises the crate's -`InvalidState`. If the action hasn't installed its cancel token yet, -`cancel_action` raises `SessionError("Cannot cancel: the action has not -started yet")`, the binding's own message: at once when called from the -action's own thread (a logging handler), and after 20 s otherwise. Every -error after acceptance, such as a parameter value that can't be coerced, -is reported on `action_status` and doesn't raise, even when it arrives -before the call returns. The crate documents those as failing the action -rather than refusing the call, and the pure-Python reference reports -them the same way. +`InvalidState`; the pure-Python reference accepts it and shortens the +grace period to its `time_limit`, which the crate can't do. If the +action hasn't installed its cancel token yet, `cancel_action` raises +`SessionError("Cannot cancel: the action has not started yet")`, the +binding's own message: at once when called from the action's own thread +(a logging handler), and after 20 s otherwise, typically because the +action thread is blocked on a lock the caller holds. Every error after +acceptance, such as a parameter value that can't be coerced, is reported +on `action_status` and doesn't raise, even when it arrives before the +call returns. The crate documents those as failing the action rather +than refusing the call, and the pure-Python reference reports them the +same way. The crate runs each action on a background thread, and that thread logs through Python's `logging` (each record needs the GIL and runs the @@ -241,7 +244,9 @@ handlers). An action method waits for the crate to accept or refuse the call, and `cancel_action` waits for the action's cancel token, so don't call either while holding a lock that a logging handler takes on another thread: the action thread would wait for the lock, and the call for the -action thread. +action thread. The pure-Python reference logs on the caller's thread, so +it has no such restriction. A fix belongs in the crate: report that it +accepted a call, and install the cancel token, before logging. An action that ends `FAILED`, `CANCELED` or `TIMEOUT` makes the session ending-only (`READY_ENDING`): `exit_environment` and `cleanup` still diff --git a/src/openjd/_openjd_rs.pyi b/src/openjd/_openjd_rs.pyi index 3322949e..3f26b57b 100644 --- a/src/openjd/_openjd_rs.pyi +++ b/src/openjd/_openjd_rs.pyi @@ -2221,6 +2221,9 @@ class Session: wrap hook is entered. A failure after that, such as an environment variable that doesn't resolve, fails the action; the environment still counts as entered and must be exited. + + Waits for the action thread, which logs through Python's `logging`, + so don't call it while holding a lock that a logging handler takes. """ def exit_environment( @@ -2239,6 +2242,9 @@ class Session: `READY_ENDING`, if `identifier` isn't entered, or if it isn't the most recently entered environment. A failure after that fails the action, and the environment still counts as exited. + + Waits for the action thread, which logs through Python's `logging`, + so don't call it while holding a lock that a logging handler takes. """ def run_task( @@ -2258,6 +2264,9 @@ class Session: failure after that, such as a task parameter value that can't be coerced, fails the action. + Waits for the action thread, which logs through Python's `logging`, + so don't call it while holding a lock that a logging handler takes. + `step_name` is surfaced as `WrappedStep.Name` to a wrapping environment's `onWrapTaskRun` hook (RFC 0008). It defaults to an empty string when the caller does not supply one. @@ -2279,6 +2288,9 @@ class Session: Raises `SessionError` if the session isn't `READY`, if `command` is empty or blank, or if `timeout` is zero, and `ValueError` if `timeout` is negative, NaN or infinite. + + Waits for the action thread, which logs through Python's `logging`, + so don't call it while holding a lock that a logging handler takes. """ def clear_ending_only(self) -> None: @@ -2307,8 +2319,12 @@ class Session: `SessionError("Cannot cancel: the action has not started yet")` if the action hasn't installed its cancel token: at once when called from the action's own thread (a logging handler), and after 20 s - otherwise, which happens only when the action thread is blocked on a - lock the caller holds, such as a logging handler's. + otherwise, typically because the action thread is blocked on a lock + the caller holds, such as a logging handler's. + + Unlike the pure-Python reference, a second cancel while one is in + progress raises; the crate doesn't shorten the first cancel's grace + period. """ def cleanup(self) -> None: diff --git a/test/openjd/sessions/test_refused_calls.py b/test/openjd/sessions/test_refused_calls.py index 5701ce18..ef3b9488 100644 --- a/test/openjd/sessions/test_refused_calls.py +++ b/test/openjd/sessions/test_refused_calls.py @@ -427,7 +427,7 @@ def test_straight_after_the_call_that_started_the_action( began = time.monotonic() session.cancel_action() _settle(session) - assert time.monotonic() - began < 4 + assert time.monotonic() - began < 8 status = session.action_status assert status is not None and status.state == ActionState.CANCELED assert session.state == SessionState.READY_ENDING @@ -521,11 +521,13 @@ def test_cleanup_with_a_concurrent_call_and_logging_on(tmp_path: Path) -> None: class CancelOnBanner(logging.Handler): def emit(self, record): if "Running Task" in record.getMessage() and not results: + began = time.monotonic() try: session.cancel_action() results.append("canceled") except SessionError as e: results.append(str(e)) + results.append(time.monotonic() - began < 0.5) logging.getLogger().addHandler(CancelOnBanner()) logging.getLogger().setLevel(logging.INFO) session = Session(session_id="s", job_parameter_values={}, session_root_directory=sys.argv[1]) @@ -555,7 +557,7 @@ def test_cancel_from_the_action_thread_before_it_starts(tmp_path: Path) -> None: timeout=25, ) assert result.returncode == 0, result.stderr - assert result.stdout.strip() == "['Cannot cancel: the action has not started yet']" + assert result.stdout.strip() == "['Cannot cancel: the action has not started yet', True]" _CANCEL_WHILE_THE_ACTION_THREAD_WAITS = r""" From 9f3a4d53d3e534745db2da469dc20a0fedc8f59e Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:41:36 +0000 Subject: [PATCH 21/21] docs: Say a second cancel in the reference shortens the grace period to at most its time limit Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- ...ions-bindings-quality-evaluation-report.md | 2 +- specs/python-sessions-interface.md | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/reports/sessions-bindings-quality-evaluation-report.md b/reports/sessions-bindings-quality-evaluation-report.md index 7cbfa43c..5bfb38e4 100644 --- a/reports/sessions-bindings-quality-evaluation-report.md +++ b/reports/sessions-bindings-quality-evaluation-report.md @@ -371,7 +371,7 @@ counted as test-suite failures. |---|---|---|---| | `run_task` while session not READY | `RuntimeError("Session must be in the READY state to run a task.")` | `SessionError("Session must be in READY state, current: RUNNING")` (the crate's message; a `RuntimeError`) | ⚠ message change | | `run_task` / `enter_environment` on a `READY_ENDING` session | `RuntimeError("Session must be in the READY state ...")` | `SessionError("Session must be in READY state, current: READY_ENDING")`, a `RuntimeError`; nothing changes | ⚠ message change (the crate's) | -| `cancel_action` while a cancel is in progress | accepted; shortens the grace period to the new `time_limit` | `SessionError("Session must be in RUNNING state, current: CANCELING")`, the crate's refusal; the crate can't shorten a cancel in progress | ⚠ behavior change | +| `cancel_action` while a cancel is in progress | accepted; shortens the remaining grace period to at most the new `time_limit` | `SessionError("Session must be in RUNNING state, current: CANCELING")`, the crate's refusal; the crate can't shorten a cancel in progress | ⚠ behavior change | | `Session.clear_ending_only()` | not in reference | takes a `READY_ENDING` session back to `READY`, as the crate's `clear_ending_only` does | ⚠ binding-only API | | `WindowsSessionUser(...)` when the process user can't be looked up | `get_process_user()` raises | raises `RuntimeError("Could not determine the process user: ...")` (before, treated as "not the process user") | ⚠ class change | | `cancel_action` when nothing is running | `RuntimeError("No actions are running")` | `SessionError("Session must be in RUNNING state, current: READY")` | ⚠ class+message change (the crate's) | diff --git a/specs/python-sessions-interface.md b/specs/python-sessions-interface.md index 361036dd..494fd810 100644 --- a/specs/python-sessions-interface.md +++ b/specs/python-sessions-interface.md @@ -226,17 +226,17 @@ one that fails before its subprocess starts can: then it raises the crate's `InvalidState`. A zero `time_limit` terminates at once. A second `cancel_action` while the session is `CANCELING` raises the crate's `InvalidState`; the pure-Python reference accepts it and shortens the -grace period to its `time_limit`, which the crate can't do. If the -action hasn't installed its cancel token yet, `cancel_action` raises -`SessionError("Cannot cancel: the action has not started yet")`, the -binding's own message: at once when called from the action's own thread -(a logging handler), and after 20 s otherwise, typically because the -action thread is blocked on a lock the caller holds. Every error after -acceptance, such as a parameter value that can't be coerced, is reported -on `action_status` and doesn't raise, even when it arrives before the -call returns. The crate documents those as failing the action rather -than refusing the call, and the pure-Python reference reports them the -same way. +remaining grace period to at most its `time_limit`, which the crate +can't do. If the action hasn't installed its cancel token yet, +`cancel_action` raises `SessionError("Cannot cancel: the action has not +started yet")`, the binding's own message: at once when called from the +action's own thread (a logging handler), and after 20 s otherwise, +typically because the action thread is blocked on a lock the caller +holds. Every error after acceptance, such as a parameter value that +can't be coerced, is reported on `action_status` and doesn't raise, even +when it arrives before the call returns. The crate documents those as +failing the action rather than refusing the call, and the pure-Python +reference reports them the same way. The crate runs each action on a background thread, and that thread logs through Python's `logging` (each record needs the GIL and runs the