Skip to content

Commit 0504a8c

Browse files
committed
fix: Stop resolving a range list at its element limit
Review of the previous commit found that an element flattening a long list (with EXPR, "{{ range_expr('1-9999999') }}" in an INT range) copied every member before the 1024-element check rejected the range: 8.4 s and 1.8 GB peak, where openjd-rs stops at that element. ListItemSpec now carries the maximum, and resolution stops at the element that takes the list past it, before its members are copied (0.04 s, 121 MB). Also from the review: - Test that text after the expression, not only before it, makes an element a string. - Test that element errors name the element's index as written when an earlier element flattened. - Assert the full element-limit message, for all five types, with the limit crossed by a scalar element, by a flattened one, and before a ten-million-member element. - Record in test_known_gaps.py that openjd.model._v1 does not flatten a list in a FLOAT range list either. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com>
1 parent bd66420 commit 0504a8c

6 files changed

Lines changed: 148 additions & 14 deletions

File tree

‎src/openjd/model/_format_strings/_format_string.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,13 @@ def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] =
236236
return self.resolve(symtab=symtab, path_format=path_format)
237237

238238
def resolve_list_item(
239-
self, *, symtab: SymbolTable, target_type: str, path_format: Optional[object] = None
240-
) -> list[Any]:
239+
self,
240+
*,
241+
symtab: SymbolTable,
242+
target_type: str,
243+
room: Optional[int] = None,
244+
path_format: Optional[object] = None,
245+
) -> Optional[list[Any]]:
241246
"""Resolve this format string as a list item (Expression Language §1.3.2).
242247
243248
When the format string is exactly one EXPR ``{{ ... }}`` expression,
@@ -246,7 +251,8 @@ def resolve_list_item(
246251
such as ``"int | list[int] | nulltype"``): a ``null`` result contributes no values, a list
247252
contributes one value per member, and any other result contributes
248253
itself. The values are the engine's ``ExprValue``s. A result that does
249-
not coerce to the target raises.
254+
not coerce to the target raises. A list longer than ``room`` (when
255+
given) returns ``None`` before its members are copied.
250256
251257
Any other format string (text around its expressions, several
252258
expressions, or a template without EXPR, whose expressions are bare
@@ -280,6 +286,8 @@ def resolve_list_item(
280286
if getattr(value, "is_null", False):
281287
return []
282288
if str(value.type).startswith("list["):
289+
if room is not None and len(value) > room:
290+
return None
283291
return list(value)
284292
return [value]
285293
return [self.resolve(symtab=symtab, path_format=path_format)]

‎src/openjd/model/_internal/_create_job.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -387,21 +387,33 @@ def _instantiate_list_items(
387387
:meth:`FormatString.resolve_list_item` resolves it to, and every value,
388388
and every element that is not a format string, goes through
389389
``spec.convert``. Errors are reported at the element's index in the list
390-
as written.
390+
as written. Resolution stops at the first element that takes the list
391+
past ``spec.max_len``, before a flattened element's members are copied.
391392
392393
Raises:
393394
ValidationError: if an element fails to resolve or convert.
394-
ValueError: if a list that was not empty resolves to no elements.
395+
ValueError: if the list resolves to more than ``spec.max_len``
396+
elements, or a list that was not empty resolves to none.
395397
"""
396398
errors: list[InitErrorDetails] = []
397399
result: list[Any] = []
398400
for idx, item in enumerate(value):
401+
resolved: Optional[list[Any]] = []
399402
with capture_validation_errors(output_errors=errors, loc=(idx,), input=value):
400403
if isinstance(item, FormatString):
401-
resolved = item.resolve_list_item(symtab=symtab, target_type=spec.target_type)
404+
resolved = item.resolve_list_item(
405+
symtab=symtab,
406+
target_type=spec.target_type,
407+
room=spec.max_len - len(result),
408+
)
402409
else:
403410
resolved = [item]
404-
result.extend(spec.convert(v) for v in resolved)
411+
if resolved is not None:
412+
result.extend(spec.convert(v) for v in resolved)
413+
# The length is a property of the whole list, so it is reported at the
414+
# field rather than at the element that went over.
415+
if resolved is None or len(result) > spec.max_len:
416+
raise ValueError(spec.too_long_message)
405417

406418
if errors:
407419
raise ValidationError.from_exception_data(

‎src/openjd/model/_types.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,10 +270,16 @@ class ListItemSpec:
270270
convert: Turns one resolved value, an ``openjd.expr.ExprValue`` from a
271271
typed element or the ``str`` of any other element, into the
272272
instantiated element. Raises ``ValueError`` to reject it.
273+
max_len: The most elements the list may resolve to. Resolution stops
274+
at the element that would take it past this, before that
275+
element's members are copied, and fails with ``too_long_message``.
276+
too_long_message: The error for a list past ``max_len``.
273277
"""
274278

275279
target_type: str
276280
convert: Callable[[Any], Any]
281+
max_len: int
282+
too_long_message: str
277283

278284

279285
@dataclass(frozen=True, eq=False, **dataclass_kwargs)

‎src/openjd/model/v2023_09/_model.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1451,14 +1451,26 @@ def _convert_string_range_list_item(value: Any) -> Any:
14511451
# list item, with target type `T? | list[T]` for element type T. A FLOAT
14521452
# element's target also accepts int, so an integer keeps the form it is
14531453
# written in, as in openjd-rs. Without EXPR every element resolves to a string.
1454+
# Resolution stops at the element that takes a list past the §3.4 maximum, so
1455+
# a template cannot make job creation build a long list just to reject it.
1456+
_RANGE_LIST_TOO_LONG = f"range exceeds {_MAX_TASK_PARAM_RANGE_LEN} elements."
14541457
_INT_RANGE_LIST_ITEM = ListItemSpec(
1455-
target_type="int | list[int] | nulltype", convert=_convert_int_range_list_item
1458+
target_type="int | list[int] | nulltype",
1459+
convert=_convert_int_range_list_item,
1460+
max_len=_MAX_TASK_PARAM_RANGE_LEN,
1461+
too_long_message=_RANGE_LIST_TOO_LONG,
14561462
)
14571463
_FLOAT_RANGE_LIST_ITEM = ListItemSpec(
1458-
target_type="float | int | list[float] | nulltype", convert=_convert_float_range_list_item
1464+
target_type="float | int | list[float] | nulltype",
1465+
convert=_convert_float_range_list_item,
1466+
max_len=_MAX_TASK_PARAM_RANGE_LEN,
1467+
too_long_message=_RANGE_LIST_TOO_LONG,
14591468
)
14601469
_STRING_RANGE_LIST_ITEM = ListItemSpec(
1461-
target_type="string | list[string] | nulltype", convert=_convert_string_range_list_item
1470+
target_type="string | list[string] | nulltype",
1471+
convert=_convert_string_range_list_item,
1472+
max_len=_MAX_TASK_PARAM_RANGE_LEN,
1473+
too_long_message=_RANGE_LIST_TOO_LONG,
14621474
)
14631475

14641476

‎test/openjd/model_v0/test_range_list_items.py‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,14 @@ def test_string_and_path_null_drops_and_list_flattens(self, tmp_path, param_type
225225
# Text around the expression, even white space, makes the element a
226226
# string rather than a list item.
227227
assert result == ["/c", "/a", "/b", " x "]
228+
result = _range(
229+
tmp_path,
230+
{"type": param_type, "range": ["{{ 'x' }} ", " {{ 'y' }}"]},
231+
[{"name": "L", "type": "LIST[STRING]"}],
232+
{"L": "[]"},
233+
["EXPR"],
234+
)
235+
assert result == ["x ", " y"]
228236

229237
@pytest.mark.parametrize(
230238
"task_param, extensions, target",
@@ -306,13 +314,59 @@ def test_list_emptied_by_null_rejected(self, tmp_path, task_param, extensions) -
306314
"has no elements after resolution."
307315
)
308316

309-
def test_flattened_list_past_element_limit_rejected(self, tmp_path) -> None:
317+
@pytest.mark.parametrize(
318+
"task_param, extensions, member, scalar",
319+
[
320+
pytest.param({"type": "INT"}, ["EXPR"], "1", "7", id="INT"),
321+
pytest.param(
322+
{"type": "CHUNK[INT]", "chunks": _CHUNKS},
323+
["EXPR", "TASK_CHUNKING"],
324+
"1",
325+
"7",
326+
id="CHUNK",
327+
),
328+
pytest.param({"type": "FLOAT"}, ["EXPR"], "1.5", "2.5", id="FLOAT"),
329+
pytest.param({"type": "STRING"}, ["EXPR"], "'a'", "b", id="STRING"),
330+
pytest.param({"type": "PATH"}, ["EXPR"], "'a'", "b", id="PATH"),
331+
],
332+
)
333+
def test_flattened_list_past_element_limit_rejected(
334+
self, tmp_path, task_param, extensions, member, scalar
335+
) -> None:
336+
full = f"{{{{ [{member}] * 1024 }}}}"
337+
assert len(
338+
_range(tmp_path, {**task_param, "range": [full]}, _STRING_S, {"S": "x"}, extensions)
339+
) == (1024)
340+
for range_ in (
341+
[scalar, full],
342+
[full, scalar],
343+
# Resolution stops at the second element, before its members are
344+
# copied; a ten-million-member list would otherwise take seconds.
345+
[full, "{{ range_expr('1-9999999') }}" if member == "1" else full],
346+
):
347+
error = _range_error(
348+
tmp_path, {**task_param, "range": range_}, _STRING_S, {"S": "x"}, extensions
349+
)
350+
assert error == (
351+
"1 validation errors for JobTemplate\n"
352+
"steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range:\n\t"
353+
"range exceeds 1024 elements."
354+
), range_
355+
356+
def test_errors_name_the_element_as_written(self, tmp_path) -> None:
357+
# The element that fails is element 2 of the list as written, even
358+
# though the first element flattened into three values.
310359
error = _range_error(
311360
tmp_path,
312-
{"type": "INT", "range": [0, "{{ range_expr('1-1024') }}"]},
361+
{"type": "INT", "range": ["{{ [1, 2, 3] }}", "{{ null }}", "{{ Param.S }}"]},
313362
_STRING_S,
314363
{"S": "x"},
315364
["EXPR"],
316365
)
317-
assert "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range:\n\t" in error
318-
assert "1024" in error
366+
assert error == (
367+
"1 validation errors for JobTemplate\n"
368+
"steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[2]:\n\t"
369+
"Failed to parse interpolation expression at [0, 13]. Expression: Param.S . "
370+
"Reason: Expression failed validation: Cannot coerce string to int | list[int] | nulltype\n"
371+
" Param.S\n ~~~~~~^."
372+
)

‎test/openjd/model_v1/test_known_gaps.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,3 +172,45 @@ def test_v1_accepts_format_string_in_int_range_list() -> None:
172172
],
173173
}
174174
)
175+
176+
177+
# With EXPR, v0 resolves a FLOAT range list element as a list item (Expression
178+
# Language §1.3.2), so a list flattens. v1's openjd-model 0.13.0 resolves it to
179+
# text and fails to parse it. Same resolution as the test above.
180+
181+
182+
@pytest.mark.xfail(strict=True, reason="bindings pin openjd-model 0.13.0 (FLOAT elements as text)")
183+
def test_v1_flattens_list_in_float_range_list() -> None:
184+
from pathlib import Path
185+
186+
import openjd.model._v1 as v1
187+
188+
template = v1.decode_job_template(
189+
{
190+
"specificationVersion": "jobtemplate-2023-09",
191+
"extensions": ["EXPR"],
192+
"name": "T",
193+
"steps": [
194+
{
195+
"name": "S",
196+
"parameterSpace": {
197+
"taskParameterDefinitions": [
198+
{"name": "X", "type": "FLOAT", "range": [1.5, "{{ [2, 2.5] }}"]}
199+
]
200+
},
201+
"script": {"actions": {"onRun": {"command": "echo"}}},
202+
}
203+
],
204+
},
205+
supported_extensions=["EXPR"],
206+
)
207+
params = v1.preprocess_job_parameters(
208+
job_template=template,
209+
job_parameter_values={},
210+
job_template_dir=Path.cwd(),
211+
current_working_dir=Path.cwd(),
212+
)
213+
job = v1.create_job(job_template=template, job_parameter_values=params)
214+
space = job.steps[0].parameter_space
215+
assert space is not None
216+
assert len(space.task_parameter_definitions["X"].range) == 3

0 commit comments

Comments
 (0)