Skip to content

Commit 085609a

Browse files
authored
feat(model): Expose chunks_task_count_override on the v1 iterator (#344)
* feat(model): Expose chunks_task_count_override on the v1 iterator Signed-off-by: David Leong <leongdl@amazon.com>
1 parent 4bfb4a3 commit 085609a

5 files changed

Lines changed: 417 additions & 6 deletions

File tree

‎rust-bindings/src/model/step_param_space.rs‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -123,14 +123,52 @@ pub(crate) struct PyStepParameterSpaceIterator {
123123
/// every `NodeIterator` impl is `Send + Sync` (enforced at the
124124
/// trait bound in `openjd-model`).
125125
iter: Mutex<StepParameterSpaceIterator>,
126+
/// The `chunks_task_count_override` this was constructed with, kept
127+
/// so `__getitem__` can rebuild an equivalent iterator. Without it,
128+
/// random access would silently fall back to the template's own
129+
/// `defaultTaskCount` and disagree with iteration.
130+
chunk_override: Option<usize>,
126131
}
127132

128133
#[cfg_attr(feature = "stub-gen", gen_stub_pymethods)]
129134
#[pymethods]
130135
impl PyStepParameterSpaceIterator {
136+
/// Construct an iterator over a step's parameter space.
137+
///
138+
/// `chunks_task_count_override` overrides the `defaultTaskCount` of a
139+
/// `CHUNK[INT]` parameter and turns adaptive chunking off, so a chunked space
140+
/// can be walked at a caller-chosen granularity. Pass `1` to iterate individual
141+
/// tasks. Ignored when the space has no chunked parameter, matching the
142+
/// pure-Python reference — though a non-positive value is still rejected in
143+
/// that case, since validating an argument is cheaper to reason about than
144+
/// silently discarding a bad one.
145+
///
146+
/// Without this, a statically chunked space could only be walked at the
147+
/// template's own chunk size: `chunks_default_task_count` is settable for
148+
/// adaptive spaces only.
131149
#[new]
132-
#[pyo3(signature = (*, step=None, space=None))]
133-
fn new(step: Option<&PyStep>, space: Option<&PyStepParameterSpace>) -> PyResult<Self> {
150+
#[pyo3(signature = (*, step=None, space=None, chunks_task_count_override=None))]
151+
fn new(
152+
step: Option<&PyStep>,
153+
space: Option<&PyStepParameterSpace>,
154+
chunks_task_count_override: Option<i64>,
155+
) -> PyResult<Self> {
156+
// Taken as i64, not usize, so a negative value reaches this check instead of
157+
// failing pyo3's unsigned extraction with OverflowError. Every non-positive
158+
// value should report the same ValueError the message and the spec promise.
159+
//
160+
// Rejected rather than clamped: openjd-model applies `.max(1)` to the override,
161+
// so 0 would silently mean 1. The `chunks_default_task_count` setter already
162+
// refuses 0, and the two should not disagree.
163+
let chunks_task_count_override: Option<usize> = match chunks_task_count_override {
164+
Some(n) if n <= 0 => {
165+
return Err(pyo3::exceptions::PyValueError::new_err(
166+
"chunks_task_count_override must be a positive integer.",
167+
));
168+
}
169+
Some(n) => Some(n as usize),
170+
None => None,
171+
};
134172
let ps = if let Some(s) = space {
135173
s.inner.clone()
136174
} else if let Some(st) = step {
@@ -148,14 +186,17 @@ impl PyStepParameterSpaceIterator {
148186
combination: None,
149187
}
150188
};
151-
let iter = StepParameterSpaceIterator::new(&ps).map_err(model_err_to_py)?;
189+
let iter =
190+
StepParameterSpaceIterator::new_with_chunk_override(&ps, chunks_task_count_override)
191+
.map_err(model_err_to_py)?;
152192
let len = iter.len();
153193
let names = iter.names().clone();
154194
Ok(Self {
155195
space: ps,
156196
len,
157197
names,
158198
iter: Mutex::new(iter),
199+
chunk_override: chunks_task_count_override,
159200
})
160201
}
161202

@@ -185,8 +226,12 @@ impl PyStepParameterSpaceIterator {
185226
index as usize
186227
};
187228
// Random access uses a fresh iterator — don't disturb the
188-
// persistent iter's cursor or its adaptive Arc.
189-
let iter = StepParameterSpaceIterator::new(&self.space).map_err(model_err_to_py)?;
229+
// persistent iter's cursor or its adaptive Arc. It must carry the
230+
// same chunk override, or indexing would report chunks that
231+
// iteration never yields.
232+
let iter =
233+
StepParameterSpaceIterator::new_with_chunk_override(&self.space, self.chunk_override)
234+
.map_err(model_err_to_py)?;
190235
match iter.get(idx) {
191236
Some(params) => task_param_set_to_py(py, &params),
192237
None => Err(pyo3::exceptions::PyIndexError::new_err(

‎specs/python-model-interface.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,8 @@ from openjd.model._v1.job import StepParameterSpaceIterator
11271127

11281128
it = StepParameterSpaceIterator(step=job.steps[0])
11291129
# or: it = StepParameterSpaceIterator(space=step.parameterSpace)
1130+
# or, to walk a chunked space one task at a time:
1131+
# it = StepParameterSpaceIterator(space=..., chunks_task_count_override=1)
11301132

11311133
len(it) # total task count, e.g. 10
11321134
it[0] # {"Frame": 1}
@@ -1150,6 +1152,45 @@ non-trivial state for chunked spaces). Indexing (``it[i]``) is
11501152
unaffected by iteration position; ``__contains__`` is also
11511153
non-mutating.
11521154

1155+
`chunks_task_count_override` replaces the `defaultTaskCount` of a
1156+
`CHUNK[INT]` parameter and turns adaptive chunking off, so a chunked
1157+
space can be walked at a granularity the caller picks. Pass `1` to
1158+
iterate individual tasks — a `1-20` range chunked five at a time then
1159+
yields `1-1`, `2-2`, … instead of `1-5`, `6-10`, …. It is ignored when
1160+
the space has no chunked parameter, matching the pure-Python reference,
1161+
although a non-positive value is still rejected in that case.
1162+
Iteration observes it, and `len()` counts the overridden granularity.
1163+
1164+
The rendered form of a chunk depends on `rangeConstraint`, which matters
1165+
because these are strings a consumer parses and may feed back through
1166+
`__contains__`. A single-task chunk is `"1-1"` under `CONTIGUOUS` and a
1167+
bare `"1"` under `NONCONTIGUOUS`.
1168+
1169+
It is currently the only way to change the chunk size of a *static*
1170+
chunked space: the `chunks_default_task_count` setter accepts adaptive
1171+
spaces only, and raises `ValueError` otherwise.
1172+
1173+
Any non-positive value raises `ValueError`, so one `except ValueError`
1174+
covers `0` and negatives alike. The Rust layer clamps the override to at
1175+
least 1, so 0 would otherwise silently mean 1, and the
1176+
`chunks_default_task_count` setter already rejects it. The pure-Python
1177+
reference does not validate this argument.
1178+
1179+
Two current-implementation limitations, both divergences from the v0
1180+
reference rather than intended behaviour. Each has a `strict` xfail in
1181+
`test/openjd/model_v1/test_known_gaps.py`, so clearing either will fail
1182+
CI until this text is updated with it.
1183+
1184+
- Indexing observes the override only for a space that supports random
1185+
access. A `CONTIGUOUS` chunked space requires sequential iteration, so
1186+
`it[i]` raises `IndexError` for any index — with or without the
1187+
override — even though `len(it)` reports a count. See
1188+
`test_a_contiguous_chunked_space_supports_indexing`.
1189+
- `chunks_parameter_name` and `chunks_default_task_count` both return
1190+
`None` once the space is non-adaptive, which supplying the override
1191+
makes it. v0 reports the parameter name and the override value. See
1192+
`test_chunk_metadata_is_reported_for_a_non_adaptive_space`.
1193+
11531194
### `StepDependencyGraph`
11541195

11551196
Step dependency graph for topological ordering.

‎src/openjd/_openjd_rs.pyi‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2381,7 +2381,23 @@ class StepParameterSpaceIterator:
23812381
*,
23822382
step: typing.Optional[Step] = None,
23832383
space: typing.Optional[StepParameterSpace] = None,
2384-
) -> StepParameterSpaceIterator: ...
2384+
chunks_task_count_override: typing.Optional[builtins.int] = None,
2385+
) -> StepParameterSpaceIterator:
2386+
r"""
2387+
Construct an iterator over a step's parameter space.
2388+
2389+
`chunks_task_count_override` overrides the `defaultTaskCount` of a
2390+
`CHUNK[INT]` parameter and turns adaptive chunking off, so a chunked space
2391+
can be walked at a caller-chosen granularity. Pass `1` to iterate individual
2392+
tasks. Ignored when the space has no chunked parameter, matching the
2393+
pure-Python reference -- though a non-positive value is still rejected in
2394+
that case.
2395+
2396+
Without this, a statically chunked space could only be walked at the
2397+
template's own chunk size: `chunks_default_task_count` is settable for
2398+
adaptive spaces only.
2399+
"""
2400+
23852401
def __len__(self) -> builtins.int: ...
23862402
def __getitem__(self, index: builtins.int) -> dict: ...
23872403
def __iter__(self) -> StepParameterSpaceIterator:

‎test/openjd/model_v1/test_known_gaps.py‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
from __future__ import annotations
2121

22+
from typing import Any
23+
2224
# ── Top-level package no longer leaks typing imports ──
2325
#
2426
# An earlier draft of ``openjd.model._v1`` imported ``Any``,
@@ -41,3 +43,121 @@ def test_no_internal_imports_leak_at_top_level(name: str) -> None:
4143
import openjd.model._v1 as v1
4244

4345
assert not hasattr(v1, name), f"{name} leaks as a public attribute on openjd.model._v1"
46+
47+
48+
# ── Chunked parameter spaces: two divergences from the v0 reference ──
49+
#
50+
# Found while adding `chunks_task_count_override` to
51+
# `StepParameterSpaceIterator`. Neither is caused by that argument — both
52+
# reproduce without it — so they are recorded here rather than fixed in
53+
# passing.
54+
55+
56+
def _chunked_step(constraint: str) -> Any:
57+
from openjd.model._v1 import create_job, decode_job_template
58+
59+
template = {
60+
"specificationVersion": "jobtemplate-2023-09",
61+
"name": "T",
62+
"extensions": ["TASK_CHUNKING"],
63+
"steps": [
64+
{
65+
"name": "S",
66+
"parameterSpace": {
67+
"taskParameterDefinitions": [
68+
{
69+
"name": "Frame",
70+
"type": "CHUNK[INT]",
71+
"range": "1-10",
72+
"chunks": {"defaultTaskCount": 5, "rangeConstraint": constraint},
73+
}
74+
]
75+
},
76+
"script": {
77+
"actions": {"onRun": {"command": "echo", "args": ["{{Task.Param.Frame}}"]}}
78+
},
79+
}
80+
],
81+
}
82+
job_template = decode_job_template(template=template, supported_extensions=["TASK_CHUNKING"])
83+
return create_job(job_template=job_template, job_parameter_values={}).steps[0]
84+
85+
86+
@pytest.mark.xfail(
87+
reason="v1 derives chunks_parameter_name and chunks_default_task_count from adaptive "
88+
"detection, so both are None for any non-adaptive chunked space. v0 reports them for "
89+
"any chunked space. See openjd-model step_param_space.rs: chunks_param_name and "
90+
"adaptive_chunk_size are both built from adaptive_info.",
91+
strict=True,
92+
)
93+
@pytest.mark.parametrize(
94+
"chunks,override,expected_count",
95+
[
96+
# Statically chunked: no override involved, v0 reports the template's size.
97+
({"defaultTaskCount": 5, "rangeConstraint": "CONTIGUOUS"}, None, 5),
98+
# Statically chunked, re-chunked by the override.
99+
({"defaultTaskCount": 5, "rangeConstraint": "CONTIGUOUS"}, 1, 1),
100+
# Adaptive, turned static by the override. v0 reports the override as the size.
101+
(
102+
{"defaultTaskCount": 5, "targetRuntimeSeconds": 60, "rangeConstraint": "CONTIGUOUS"},
103+
1,
104+
1,
105+
),
106+
],
107+
ids=["static", "static-overridden", "adaptive-overridden"],
108+
)
109+
def test_chunk_metadata_is_reported_for_a_non_adaptive_space(
110+
chunks: dict, override: int | None, expected_count: int
111+
) -> None:
112+
"""v0 returns ``"Frame"`` and the chunk size for each of these. v1 returns ``None``.
113+
114+
Neither value is unknowable — both are in the template, or are the override the caller
115+
just passed — so a consumer inspecting a non-adaptive chunked space through v1 cannot
116+
learn which parameter chunks, or at what size. One root cause, three ways to reach it:
117+
anything that leaves the space non-adaptive drops both getters.
118+
"""
119+
from openjd.model._v1 import create_job, decode_job_template
120+
from openjd.model._v1.job import StepParameterSpaceIterator
121+
122+
template = {
123+
"specificationVersion": "jobtemplate-2023-09",
124+
"name": "T",
125+
"extensions": ["TASK_CHUNKING"],
126+
"steps": [
127+
{
128+
"name": "S",
129+
"parameterSpace": {
130+
"taskParameterDefinitions": [
131+
{"name": "Frame", "type": "CHUNK[INT]", "range": "1-10", "chunks": chunks}
132+
]
133+
},
134+
"script": {
135+
"actions": {"onRun": {"command": "echo", "args": ["{{Task.Param.Frame}}"]}}
136+
},
137+
}
138+
],
139+
}
140+
job_template = decode_job_template(template=template, supported_extensions=["TASK_CHUNKING"])
141+
step = create_job(job_template=job_template, job_parameter_values={}).steps[0]
142+
143+
it = StepParameterSpaceIterator(step=step, chunks_task_count_override=override)
144+
assert it.chunks_adaptive is False
145+
assert it.chunks_parameter_name == "Frame"
146+
assert it.chunks_default_task_count == expected_count
147+
148+
149+
@pytest.mark.xfail(
150+
reason="v1 refuses random access whenever the space needs sequential iteration, and "
151+
"contiguous chunking always does. v0 supports indexing the same space.",
152+
strict=True,
153+
)
154+
def test_a_contiguous_chunked_space_supports_indexing() -> None:
155+
"""v0 answers ``it[0]`` with ``1-5``. v1 raises ``IndexError``.
156+
157+
``len()`` works on this space, so the count is known; only ``get`` declines.
158+
"""
159+
from openjd.model._v1.job import StepParameterSpaceIterator
160+
161+
it = StepParameterSpaceIterator(step=_chunked_step("CONTIGUOUS"))
162+
assert len(it) == 2
163+
assert it[0]["Frame"].value == "1-5"

0 commit comments

Comments
 (0)