From bd66420cdefa28b581933242a12fcebcbc50dc3b Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:13:25 +0000 Subject: [PATCH 1/4] fix!: Resolve task parameter range list elements as list items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Template Schemas §3.4.1.1 and §3.4.1.2 require a element of an INT, CHUNK[INT] or FLOAT range list to resolve to a number, but job creation kept any resolved text as the value: an INT parameter could take the value "abc", and a CHUNK[INT] one raised an uncaught ValueError when its parameter space was iterated. Such an element is now rejected at job creation, at range[k], as "value 'abc' is not an integer." (or "is not a number." / "is not finite." for FLOAT). With the EXPR extension, Expression Language §1.3.2 makes every range list element a list item, for all five task parameter types. An element that is exactly one {{ ... }} expression is evaluated with the target type T? | list[T]: null drops the element, a list flattens into one element per member, and a value that does not coerce to the element type is rejected. A range emptied by dropped elements is rejected with "has no elements after resolution." An element with text around its expression still resolves to a string, as does every element without EXPR. This matches openjd-rs. - FormatString.resolve_list_item and JobCreationMetadata.list_item_fields (ListItemSpec) implement the list-item resolution; the expression nodes accept a target type. openjd.expr is still loaded only on use. - test_range_list_items.py covers both modes; test_known_gaps.py records that openjd.model._v1 still rejects INT range format strings until the bindings move to an openjd-model release with the fix. BREAKING CHANGE: with EXPR, a STRING or PATH range element that evaluates to a list now gives one task per member instead of one task whose value is the list's text, and an element that evaluates to null is dropped instead of giving an empty string. An INT, CHUNK[INT] or FLOAT element that resolves to text that is not a number now fails job creation. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- .../model/_format_strings/_expression.py | 21 +- .../model/_format_strings/_format_string.py | 60 ++++ src/openjd/model/_format_strings/_nodes.py | 19 +- src/openjd/model/_internal/_create_job.py | 50 ++- src/openjd/model/_types.py | 31 ++ src/openjd/model/v2023_09/_model.py | 68 ++++ test/openjd/model_v0/test_range_list_items.py | 318 ++++++++++++++++++ test/openjd/model_v1/test_known_gaps.py | 35 ++ 8 files changed, 595 insertions(+), 7 deletions(-) create mode 100644 test/openjd/model_v0/test_range_list_items.py diff --git a/src/openjd/model/_format_strings/_expression.py b/src/openjd/model/_format_strings/_expression.py index 844dc7c0..81681b15 100644 --- a/src/openjd/model/_format_strings/_expression.py +++ b/src/openjd/model/_format_strings/_expression.py @@ -86,9 +86,24 @@ def evaluate( raise ExpressionError(f"Nonvalid result type: {result} of type {type(result)}") - def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = None) -> Any: + @property + def is_expr(self) -> bool: + """Whether this expression uses the EXPR language (Expression Language), + rather than the base 2023-09 value-reference grammar.""" + return getattr(self._expresion_tree, "allows_nonscalar", False) + + def evaluate_value( + self, + *, + symtab: SymbolTable, + path_format: Optional[Any] = None, + target_type: Optional[Any] = None, + ) -> Any: """Evaluate the expression, preferring the engine's typed value form. + ``target_type`` (an ``openjd.expr.ExprType``), when given, applies to + EXPR-backed expressions: the result is coerced toward it. + For EXPR-backed expressions this returns the engine ``ExprValue``, which keeps the result's EXPR type (path/int/float/list/...) and its string-rendering fidelity when re-seeded into a symbol table — the @@ -103,6 +118,10 @@ def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = No if evaluate_value is None: return self.evaluate(symtab=symtab, path_format=path_format) try: + if target_type is not None: + return evaluate_value( + symtab=symtab, path_format=path_format, target_type=target_type + ) return evaluate_value(symtab=symtab, path_format=path_format) except ValueError as exc: raise ExpressionError(f"Expression failed validation: {str(exc)}") diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index b47245e2..8b3ea03c 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import functools from dataclasses import dataclass from typing import Any, Optional, Union @@ -10,6 +11,16 @@ from .._types import ModelParsingContextInterface, SpecificationRevision +@functools.lru_cache(maxsize=None) +def _expr_type(type_string: str) -> Any: + """The ``openjd.expr.ExprType`` for ``type_string``. Imported here, on + first use, because importing openjd.model must not load the native + extension; only an EXPR template reaches the callers.""" + from openjd.expr import ExprType + + return ExprType(type_string) + + class _ReconstructionContext(ModelParsingContextInterface): """Minimal parsing context used when a FormatString is reconstructed by pickle or the copy module (see ``FormatString.__getnewargs_ex__``). @@ -224,6 +235,55 @@ def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] = ) return self.resolve(symtab=symtab, path_format=path_format) + def resolve_list_item( + self, *, symtab: SymbolTable, target_type: str, path_format: Optional[object] = None + ) -> list[Any]: + """Resolve this format string as a list item (Expression Language §1.3.2). + + When the format string is exactly one EXPR ``{{ ... }}`` expression, + with no text around it, the expression is evaluated with + ``target_type`` (an EXPR type string of the form ``T? | list[T]``, + such as ``"int | list[int] | nulltype"``): a ``null`` result contributes no values, a list + contributes one value per member, and any other result contributes + itself. The values are the engine's ``ExprValue``s. A result that does + not coerce to the target raises. + + Any other format string (text around its expressions, several + expressions, or a template without EXPR, whose expressions are bare + value references that cannot produce ``null`` or a list) contributes + its resolved string, as :meth:`resolve` returns it. + + Raises: + FormatStringError: if the expression cannot be evaluated or its + result does not coerce to ``target_type``. + """ + expressions = self.expressions + if len(expressions) == 1: + info = expressions[0] + expression = info.expression + whole_field = info.start_pos == 0 and info.end_pos == len(self.original_value) + if whole_field and expression is not None and getattr(expression, "is_expr", False): + try: + value = expression.evaluate_value( + symtab=symtab, + path_format=path_format, + target_type=_expr_type(target_type), + ) + except ExpressionError as exc: + raise FormatStringError( + string=self.original_value, + start=info.start_pos, + end=info.end_pos, + expr=expression.expr, + details=str(exc), + ) + if getattr(value, "is_null", False): + return [] + if str(value.type).startswith("list["): + return list(value) + return [value] + return [self.resolve(symtab=symtab, path_format=path_format)] + def _preprocess( self, *, context: ModelParsingContextInterface ) -> list[Union[str, ExpressionInfo]]: diff --git a/src/openjd/model/_format_strings/_nodes.py b/src/openjd/model/_format_strings/_nodes.py index 750283b6..5f760efb 100644 --- a/src/openjd/model/_format_strings/_nodes.py +++ b/src/openjd/model/_format_strings/_nodes.py @@ -220,12 +220,17 @@ def _check_comprehension_shadowing(self, symbols: set) -> None: "a variable that already exists at this location." ) - def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: + def _evaluate_raw( + self, *, symtab: SymbolTable, path_format: Any = None, target_type: Any = None + ) -> Any: """Evaluate the expression and return the engine's ``ExprValue``. Shared by :meth:`evaluate` (which unwraps to the native Python value) and :meth:`evaluate_to_str` (which uses the engine's spec-defined string coercion). Re-raises engine errors as the model's ``ExpressionError``. + ``target_type`` (an ``openjd.expr.ExprType``), when given, is the + result's target type (Expression Language §1.2.3): the result is + coerced toward it, and evaluation fails when it cannot be. """ from ._expr_support import ( map_eval_error, @@ -246,16 +251,20 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: # is not absolute, and that reaches the caller as an expression # error like any other. profile = profile_for_symtab(symtab) - return self._parsed.evaluate(values=values, profile=profile, path_format=path_format) + return self._parsed.evaluate( + values=values, profile=profile, path_format=path_format, target_type=target_type + ) except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error raise map_eval_error(exc) def evaluate(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: return self._evaluate_raw(symtab=symtab, path_format=path_format).item() - def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: + def evaluate_value( + self, *, symtab: SymbolTable, path_format: Any = None, target_type: Any = None + ) -> Any: """Evaluate and return the engine's ``ExprValue`` (not unwrapped to a - native Python value). + native Python value), coerced toward ``target_type`` when given. Callers that re-seed the result into a symbol table (e.g. the session runtime's ``let`` bindings, RFC 0007) should use this form: the typed @@ -265,7 +274,7 @@ def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any are preserved), matching the Rust runtime's natively typed symbol table. """ - return self._evaluate_raw(symtab=symtab, path_format=path_format) + return self._evaluate_raw(symtab=symtab, path_format=path_format, target_type=target_type) def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Any = None) -> str: # ``str(ExprValue)`` applies the engine's RFC 0005 format-string diff --git a/src/openjd/model/_internal/_create_job.py b/src/openjd/model/_internal/_create_job.py index f5b47f3c..4c919d14 100644 --- a/src/openjd/model/_internal/_create_job.py +++ b/src/openjd/model/_internal/_create_job.py @@ -9,7 +9,7 @@ from .._symbol_table import SymbolTable from .._format_strings import FormatString -from .._types import ModelParsingContextInterface, OpenJDModel +from .._types import ListItemSpec, ModelParsingContextInterface, OpenJDModel __all__ = ("instantiate_model", "resolve_whole_field_typed_list") @@ -262,6 +262,15 @@ def _instantiate_model( # noqa: C901 instantiated = coerce(model, field_name, raw_typed) else: instantiated = unwrap_typed_resolution(raw_typed) + elif ( + isinstance(field_value, list) + and field_name in model._job_creation_metadata.list_item_fields + ): + instantiated = _instantiate_list_items( + field_value, + symtab, + model._job_creation_metadata.list_item_fields[field_name], + ) elif isinstance(field_value, list): if field_name in model._job_creation_metadata.reshape_field_to_dict: key_field = model._job_creation_metadata.reshape_field_to_dict[field_name] @@ -366,6 +375,45 @@ def _instantiate_list_field_as_list( # noqa: C901 return result +def _instantiate_list_items( + value: list[Any], + symtab: SymbolTable, + spec: ListItemSpec, +) -> list[Any]: + """Instantiate a list whose format string elements are list items + (Expression Language §1.3.2; see ``JobCreationMetadata.list_item_fields``). + + Each format string element contributes the values + :meth:`FormatString.resolve_list_item` resolves it to, and every value, + and every element that is not a format string, goes through + ``spec.convert``. Errors are reported at the element's index in the list + as written. + + Raises: + ValidationError: if an element fails to resolve or convert. + ValueError: if a list that was not empty resolves to no elements. + """ + errors: list[InitErrorDetails] = [] + result: list[Any] = [] + for idx, item in enumerate(value): + with capture_validation_errors(output_errors=errors, loc=(idx,), input=value): + if isinstance(item, FormatString): + resolved = item.resolve_list_item(symtab=symtab, target_type=spec.target_type) + else: + resolved = [item] + result.extend(spec.convert(v) for v in resolved) + + if errors: + raise ValidationError.from_exception_data( + title="_instantiate_list_items", line_errors=errors + ) + # The non-empty rule is checked on the list as written, at decode; dropped + # elements can still leave it empty. + if value and not result: + raise ValueError("has no elements after resolution.") + return result + + def _instantiate_list_field_as_dict( # noqa: C901 value: list[Any], symtab: SymbolTable, diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index e4174f66..d1c5cdf1 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -258,6 +258,24 @@ class JobCreateAsMetadata: callable: Optional[Callable[["OpenJDModel", dict], Type["OpenJDModel"]]] = field(default=None) +@dataclass(frozen=True, eq=False, **dataclass_kwargs) +class ListItemSpec: + """How the elements of a list field resolve as list items (see + ``JobCreationMetadata.list_item_fields``). + + Attributes: + target_type: The EXPR type string of a whole-field EXPR element's + target, of the form ``T? | list[T]``, such as + ``"int | list[int] | nulltype"``. + convert: Turns one resolved value, an ``openjd.expr.ExprValue`` from a + typed element or the ``str`` of any other element, into the + instantiated element. Raises ``ValueError`` to reject it. + """ + + target_type: str + convert: Callable[[Any], Any] + + @dataclass(frozen=True, eq=False, **dataclass_kwargs) class JobCreationMetadata: """Metadata for creating a job out of a job template. @@ -302,6 +320,19 @@ class JobCreationMetadata: native Python list unchanged. """ + list_item_fields: dict[str, "ListItemSpec"] = field(default_factory=dict) + """List fields (a subset of ``resolve_fields``) whose format string + elements are list items (Expression Language §1.3.2), keyed by field name. + Each element resolves through :meth:`FormatString.resolve_list_item` with + the spec's target type, so with EXPR a whole-field ``null`` drops the + element and a list flattens into one element per member. The spec's + ``convert`` turns each resolved value into the instantiated element, and + may raise ``ValueError`` to reject it; the error is reported at the + element's index in the list as written. A list that was not empty as + written but resolves to no elements is rejected. Used by task-parameter + ``range`` lists. + """ + create_as: Optional[JobCreateAsMetadata] = field(default=None) """The class to create the model as during instantiation. Options: 1. None -- create as the model class itself. diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 13a3a286..415b080a 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import re import secrets import string @@ -55,6 +56,7 @@ JobCreateAsMetadata, JobCreationMetadata, JobParameterInterface, + ListItemSpec, ModelParsingContextInterface, OpenJDModel, ResolutionScope, @@ -1399,6 +1401,67 @@ def _range_element_type_name(elem: Any) -> str: return "list" if type_str.startswith("list[") else type_str +def _expr_value_type(value: Any) -> Optional[str]: + """The EXPR type of an engine ``ExprValue``, or ``None`` for a native value.""" + value_type = getattr(value, "type", None) + return None if value_type is None else str(value_type) + + +def _convert_int_range_list_item(value: Any) -> int: + """One INT or CHUNK[INT] range list element (Template Schemas §3.4.1.1), + from a literal, a typed list item, or a resolved format string, which + must be the string representation of an integer.""" + if _expr_value_type(value) == "int": + return value.item() + if isinstance(value, str): + try: + return int(value) + except ValueError: + raise ValueError(f"value {value!r} is not an integer.") + return value + + +def _convert_float_range_list_item(value: Any) -> Any: + """One FLOAT range list element (Template Schemas §3.4.1.2). A typed list + item becomes its string form, which keeps a float's spelling and writes an + integer as written (openjd-rs does the same), and a resolved string must + be the string representation of a finite floating point value. Strings + are normalized by ``RangeListTaskParameterDefinition`` (§7.5).""" + if _expr_value_type(value) is not None: + value = str(value) + if isinstance(value, str): + try: + number = float(value) + except ValueError: + raise ValueError(f"value {value!r} is not a number.") + if not math.isfinite(number): + raise ValueError(f"value {value!r} is not finite.") + return value + + +def _convert_string_range_list_item(value: Any) -> Any: + """One STRING or PATH range list element: a typed list item becomes its + string form (Expression Language string coercion).""" + if _expr_value_type(value) is not None: + return str(value) + return value + + +# Expression Language §1.3.2: each element of a task parameter range list is a +# list item, with target type `T? | list[T]` for element type T. A FLOAT +# element's target also accepts int, so an integer keeps the form it is +# written in, as in openjd-rs. Without EXPR every element resolves to a string. +_INT_RANGE_LIST_ITEM = ListItemSpec( + target_type="int | list[int] | nulltype", convert=_convert_int_range_list_item +) +_FLOAT_RANGE_LIST_ITEM = ListItemSpec( + target_type="float | int | list[float] | nulltype", convert=_convert_float_range_list_item +) +_STRING_RANGE_LIST_ITEM = ListItemSpec( + target_type="string | list[string] | nulltype", convert=_convert_string_range_list_item +) + + def _coerce_typed_range_elements(model: Any, field_name: str, raw: Any) -> list: """``typed_resolve_coerce`` hook for task-parameter ``range`` fields. @@ -1521,6 +1584,7 @@ class IntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09 resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _INT_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) @@ -1589,6 +1653,7 @@ class FloatTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_ resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _FLOAT_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) @@ -1635,6 +1700,7 @@ class StringTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023 resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _STRING_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) @@ -1670,6 +1736,7 @@ class PathTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_0 resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _STRING_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) @@ -1725,6 +1792,7 @@ class ChunkIntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v20 resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _INT_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) diff --git a/test/openjd/model_v0/test_range_list_items.py b/test/openjd/model_v0/test_range_list_items.py new file mode 100644 index 00000000..55286848 --- /dev/null +++ b/test/openjd/model_v0/test_range_list_items.py @@ -0,0 +1,318 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""Job creation for the format string elements of task parameter range lists. + +Template Schemas §3.4.1.1 and §3.4.1.2: an element of an INT, CHUNK[INT] or +FLOAT range list may be a , which must resolve to +the string representation of a number. With the EXPR extension, Expression +Language §1.3.2 makes every range list element a list item, with target type +``T? | list[T]``: a whole-field ``null`` drops the element, a list flattens +into one element per member, and a value that does not coerce is rejected. +openjd-rs (crates/openjd-model/src/job/create_job/ranges.rs) does the same. +""" + +from decimal import Decimal +from pathlib import Path +from typing import Any, Optional + +import pytest + +from openjd.model import ( + DecodeValidationError, + create_job, + decode_job_template, + preprocess_job_parameters, +) + + +def _range( + tmp_path: Path, + task_param: dict[str, Any], + params: list[dict[str, Any]], + values: dict[str, str], + extensions: Optional[list[str]] = None, +) -> list[Any]: + """Create a job with one task parameter, X, and return its resolved range.""" + template: dict[str, Any] = { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": params, + "steps": [ + { + "name": "S", + "parameterSpace": {"taskParameterDefinitions": [{"name": "X", **task_param}]}, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + } + if extensions: + template["extensions"] = extensions + job_template = decode_job_template( + template=template, supported_extensions=["EXPR", "TASK_CHUNKING"] + ) + processed = preprocess_job_parameters( + job_template=job_template, + job_parameter_values=values, + job_template_dir=tmp_path, + current_working_dir=tmp_path, + ) + job = create_job(job_template=job_template, job_parameter_values=processed) + assert job.steps[0].parameterSpace is not None + return list(job.steps[0].parameterSpace.taskParameterDefinitions["X"].range) + + +def _range_error(tmp_path: Path, *args: Any, **kwargs: Any) -> str: + with pytest.raises(DecodeValidationError) as excinfo: + _range(tmp_path, *args, **kwargs) + return str(excinfo.value) + + +_CHUNKS = {"defaultTaskCount": 1, "rangeConstraint": "NONCONTIGUOUS"} +_STRING_S = [{"name": "S", "type": "STRING"}] +_ELEMENT_1 = ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[1]:\n\t" +) + + +class TestWithoutExpr: + """Without EXPR an element resolves to a string, parsed as an + or .""" + + @pytest.mark.parametrize( + "task_param, extensions", + [ + pytest.param({"type": "INT"}, None, id="INT"), + pytest.param({"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["TASK_CHUNKING"], id="CHUNK"), + ], + ) + def test_int_elements_resolve(self, tmp_path, task_param, extensions) -> None: + params = [{"name": "I", "type": "INT"}, {"name": "S", "type": "STRING"}] + result = _range( + tmp_path, + {**task_param, "range": [1, "2", "{{Param.I}}", "{{Param.S}}", "1{{Param.I}}"]}, + params, + {"I": "7", "S": " 9 "}, + extensions, + ) + assert result == [1, 2, 7, 9, 17] + + @pytest.mark.parametrize( + "task_param, extensions", + [ + pytest.param({"type": "INT"}, None, id="INT"), + # CHUNK[INT] used to keep the text and raise an uncaught ValueError + # when the parameter space was iterated. + pytest.param({"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["TASK_CHUNKING"], id="CHUNK"), + ], + ) + @pytest.mark.parametrize("value", ["abc", "2.5", ""]) + def test_int_element_not_an_integer(self, tmp_path, task_param, extensions, value) -> None: + error = _range_error( + tmp_path, + {**task_param, "range": [1, "{{Param.S}}"]}, + _STRING_S, + {"S": value}, + extensions, + ) + assert error == _ELEMENT_1 + f"value {value!r} is not an integer." + + def test_float_elements_resolve_and_keep_their_spelling(self, tmp_path) -> None: + params = [{"name": "F", "type": "FLOAT"}, {"name": "I", "type": "INT"}] + result = _range( + tmp_path, + {"type": "FLOAT", "range": [1.5, "{{Param.F}}", "{{Param.I}}", " 02.50 "]}, + params, + {"F": "2.50", "I": "10"}, + ) + assert result == [Decimal("1.5"), "2.50", "10", "2.50"] + + @pytest.mark.parametrize( + "value, problem", + [ + ("abc", "is not a number"), + ("", "is not a number"), + ("inf", "is not finite"), + ("nan", "is not finite"), + ], + ) + def test_float_element_not_a_finite_number(self, tmp_path, value, problem) -> None: + error = _range_error( + tmp_path, {"type": "FLOAT", "range": [1.5, "{{Param.S}}"]}, _STRING_S, {"S": value} + ) + assert error == _ELEMENT_1 + f"value {value!r} {problem}." + + def test_string_elements_are_not_list_items(self, tmp_path) -> None: + # Without EXPR an element cannot produce null or a list, and resolves + # to its string unchanged. + result = _range( + tmp_path, {"type": "STRING", "range": ["a", " {{Param.S}} "]}, _STRING_S, {"S": "b"} + ) + assert result == ["a", " b "] + + +class TestWithExpr: + """With EXPR a whole-field element is a list item (Expression Language §1.3.2).""" + + @pytest.mark.parametrize( + "task_param, extensions", + [ + pytest.param({"type": "INT"}, ["EXPR"], id="INT"), + pytest.param( + {"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["EXPR", "TASK_CHUNKING"], id="CHUNK" + ), + ], + ) + def test_int_null_drops_list_flattens_and_coerces( + self, tmp_path, task_param, extensions + ) -> None: + params = [{"name": "L", "type": "LIST[INT]"}, {"name": "S", "type": "STRING"}] + result = _range( + tmp_path, + { + **task_param, + "range": [ + 1, + "{{ null }}", + "{{ Param.L }}", + "{{ range_expr('7-8') }}", + "{{ 3.0 * 3 }}", + "{{ Param.S }}", + ], + }, + params, + {"L": "[3, 4]", "S": "5"}, + extensions, + ) + assert result == [1, 3, 4, 7, 8, 9, 5] + + def test_float_null_drops_list_flattens_and_keeps_spelling(self, tmp_path) -> None: + params = [ + {"name": "F", "type": "FLOAT"}, + {"name": "I", "type": "INT"}, + {"name": "L", "type": "LIST[INT]"}, + ] + result = _range( + tmp_path, + { + "type": "FLOAT", + "range": [ + 1.5, + "{{ null }}", + "{{ Param.F }}", + "{{ Param.I }}", + "{{ [Param.F] }}", + "{{ Param.L }}", + ], + }, + params, + {"F": "2.50", "I": "10", "L": "[3, 4]"}, + ["EXPR"], + ) + # A float keeps its spelling, an integer is written as is, and the + # integer members of a list converted to float are written as floats, + # as in openjd-rs. + assert result == [Decimal("1.5"), "2.50", "10", "2.50", "3.0", "4.0"] + + @pytest.mark.parametrize("param_type", ["STRING", "PATH"]) + def test_string_and_path_null_drops_and_list_flattens(self, tmp_path, param_type) -> None: + result = _range( + tmp_path, + {"type": param_type, "range": ["/c", "{{ null }}", "{{ Param.L }}", " {{ 'x' }} "]}, + [{"name": "L", "type": "LIST[STRING]"}], + {"L": '["/a", "/b"]'}, + ["EXPR"], + ) + # Text around the expression, even white space, makes the element a + # string rather than a list item. + assert result == ["/c", "/a", "/b", " x "] + + @pytest.mark.parametrize( + "task_param, extensions, target", + [ + pytest.param({"type": "INT"}, ["EXPR"], "int | list[int] | nulltype", id="INT"), + pytest.param( + {"type": "CHUNK[INT]", "chunks": _CHUNKS}, + ["EXPR", "TASK_CHUNKING"], + "int | list[int] | nulltype", + id="CHUNK", + ), + pytest.param( + {"type": "FLOAT"}, ["EXPR"], "float | int | list[float] | nulltype", id="FLOAT" + ), + ], + ) + def test_list_of_wrong_type_rejected(self, tmp_path, task_param, extensions, target) -> None: + error = _range_error( + tmp_path, + {**task_param, "range": [1, "{{ Param.L }}"]}, + [{"name": "L", "type": "LIST[STRING]"}], + {"L": '["a"]'}, + extensions, + ) + assert error == _ELEMENT_1 + ( + "Failed to parse interpolation expression at [0, 13]. Expression: Param.L . " + f"Reason: Expression failed validation: Cannot coerce list[string] to {target}\n" + " Param.L\n ~~~~~~^." + ) + + def test_padded_string_does_not_coerce(self, tmp_path) -> None: + # Expression Language §1.2.3 coercion does not trim, unlike the + # parse without EXPR. + error = _range_error( + tmp_path, + {"type": "INT", "range": [1, "{{Param.S}}"]}, + _STRING_S, + {"S": " 7 "}, + ["EXPR"], + ) + assert error == _ELEMENT_1 + ( + "Failed to parse interpolation expression at [0, 11]. Expression: Param.S. " + "Reason: Expression failed validation: Cannot coerce string to int | list[int] | nulltype\n" + " Param.S\n ~~~~~~^." + ) + + def test_text_around_expression_resolves_to_string(self, tmp_path) -> None: + result = _range( + tmp_path, + {"type": "INT", "range": ["1{{ Param.S }}", "{{ 2 }}{{ 3 }}"]}, + _STRING_S, + {"S": "5"}, + ["EXPR"], + ) + assert result == [15, 23] + + @pytest.mark.parametrize( + "task_param, extensions", + [ + pytest.param({"type": "INT"}, ["EXPR"], id="INT"), + pytest.param({"type": "FLOAT"}, ["EXPR"], id="FLOAT"), + pytest.param({"type": "STRING"}, ["EXPR"], id="STRING"), + pytest.param( + {"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["EXPR", "TASK_CHUNKING"], id="CHUNK" + ), + ], + ) + def test_list_emptied_by_null_rejected(self, tmp_path, task_param, extensions) -> None: + error = _range_error( + tmp_path, + {**task_param, "range": ["{{ null }}", "{{ null if Param.S == 'x' else 1 }}"]}, + _STRING_S, + {"S": "x"}, + extensions, + ) + assert error == ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range:\n\t" + "has no elements after resolution." + ) + + def test_flattened_list_past_element_limit_rejected(self, tmp_path) -> None: + error = _range_error( + tmp_path, + {"type": "INT", "range": [0, "{{ range_expr('1-1024') }}"]}, + _STRING_S, + {"S": "x"}, + ["EXPR"], + ) + assert "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range:\n\t" in error + assert "1024" in error diff --git a/test/openjd/model_v1/test_known_gaps.py b/test/openjd/model_v1/test_known_gaps.py index 9966817d..a40e7a13 100644 --- a/test/openjd/model_v1/test_known_gaps.py +++ b/test/openjd/model_v1/test_known_gaps.py @@ -137,3 +137,38 @@ def test_v0_range_string_renders_in_canonical_form(text: str, expected: str) -> from openjd.model._range_expr import IntRangeExpr assert str(IntRangeExpr.from_str(text)) == expected + + +# ── v1 rejects format string elements in INT range lists ── +# +# Template Schemas §3.4.1.1 allows a element in an +# INT (and CHUNK[INT]) range list, and v0 accepts one. The bindings build +# against openjd-model 0.13.0, whose `IntRange::List` holds integers only, so v1 +# fails to decode the template. The openjd-rs fix changes `IntRange::List` to +# `Vec`; this resolves when the bindings move to an +# openjd-model release with it, and `int_range_to_py` handles the new element +# type. ``test_range_list_items.py`` in model_v0 covers the v0 behavior. + + +@pytest.mark.xfail(strict=True, reason="bindings pin openjd-model 0.13.0 (IntRange::List of i64)") +def test_v1_accepts_format_string_in_int_range_list() -> None: + import openjd.model._v1 as v1 + + v1.decode_job_template( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": [{"name": "I", "type": "INT", "default": 10}], + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "X", "type": "INT", "range": [1, "{{Param.I}}"]} + ] + }, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + } + ) From 0504a8c7e34e84775d964f210f31fb6c33439786 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:50:23 +0000 Subject: [PATCH 2/4] fix: Stop resolving a range list at its element limit Review of the previous commit found that an element flattening a long list (with EXPR, "{{ range_expr('1-9999999') }}" in an INT range) copied every member before the 1024-element check rejected the range: 8.4 s and 1.8 GB peak, where openjd-rs stops at that element. ListItemSpec now carries the maximum, and resolution stops at the element that takes the list past it, before its members are copied (0.04 s, 121 MB). Also from the review: - Test that text after the expression, not only before it, makes an element a string. - Test that element errors name the element's index as written when an earlier element flattened. - Assert the full element-limit message, for all five types, with the limit crossed by a scalar element, by a flattened one, and before a ten-million-member element. - Record in test_known_gaps.py that openjd.model._v1 does not flatten a list in a FLOAT range list either. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- .../model/_format_strings/_format_string.py | 14 ++++- src/openjd/model/_internal/_create_job.py | 20 ++++-- src/openjd/model/_types.py | 6 ++ src/openjd/model/v2023_09/_model.py | 18 +++++- test/openjd/model_v0/test_range_list_items.py | 62 +++++++++++++++++-- test/openjd/model_v1/test_known_gaps.py | 42 +++++++++++++ 6 files changed, 148 insertions(+), 14 deletions(-) diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index 8b3ea03c..ad999616 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -236,8 +236,13 @@ def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] = return self.resolve(symtab=symtab, path_format=path_format) def resolve_list_item( - self, *, symtab: SymbolTable, target_type: str, path_format: Optional[object] = None - ) -> list[Any]: + self, + *, + symtab: SymbolTable, + target_type: str, + room: Optional[int] = None, + path_format: Optional[object] = None, + ) -> Optional[list[Any]]: """Resolve this format string as a list item (Expression Language §1.3.2). When the format string is exactly one EXPR ``{{ ... }}`` expression, @@ -246,7 +251,8 @@ def resolve_list_item( such as ``"int | list[int] | nulltype"``): a ``null`` result contributes no values, a list contributes one value per member, and any other result contributes itself. The values are the engine's ``ExprValue``s. A result that does - not coerce to the target raises. + not coerce to the target raises. A list longer than ``room`` (when + given) returns ``None`` before its members are copied. Any other format string (text around its expressions, several expressions, or a template without EXPR, whose expressions are bare @@ -280,6 +286,8 @@ def resolve_list_item( if getattr(value, "is_null", False): return [] if str(value.type).startswith("list["): + if room is not None and len(value) > room: + return None return list(value) return [value] return [self.resolve(symtab=symtab, path_format=path_format)] diff --git a/src/openjd/model/_internal/_create_job.py b/src/openjd/model/_internal/_create_job.py index 4c919d14..bce6b514 100644 --- a/src/openjd/model/_internal/_create_job.py +++ b/src/openjd/model/_internal/_create_job.py @@ -387,21 +387,33 @@ def _instantiate_list_items( :meth:`FormatString.resolve_list_item` resolves it to, and every value, and every element that is not a format string, goes through ``spec.convert``. Errors are reported at the element's index in the list - as written. + as written. Resolution stops at the first element that takes the list + past ``spec.max_len``, before a flattened element's members are copied. Raises: ValidationError: if an element fails to resolve or convert. - ValueError: if a list that was not empty resolves to no elements. + ValueError: if the list resolves to more than ``spec.max_len`` + elements, or a list that was not empty resolves to none. """ errors: list[InitErrorDetails] = [] result: list[Any] = [] for idx, item in enumerate(value): + resolved: Optional[list[Any]] = [] with capture_validation_errors(output_errors=errors, loc=(idx,), input=value): if isinstance(item, FormatString): - resolved = item.resolve_list_item(symtab=symtab, target_type=spec.target_type) + resolved = item.resolve_list_item( + symtab=symtab, + target_type=spec.target_type, + room=spec.max_len - len(result), + ) else: resolved = [item] - result.extend(spec.convert(v) for v in resolved) + if resolved is not None: + result.extend(spec.convert(v) for v in resolved) + # The length is a property of the whole list, so it is reported at the + # field rather than at the element that went over. + if resolved is None or len(result) > spec.max_len: + raise ValueError(spec.too_long_message) if errors: raise ValidationError.from_exception_data( diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index d1c5cdf1..f0be2d2b 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -270,10 +270,16 @@ class ListItemSpec: convert: Turns one resolved value, an ``openjd.expr.ExprValue`` from a typed element or the ``str`` of any other element, into the instantiated element. Raises ``ValueError`` to reject it. + max_len: The most elements the list may resolve to. Resolution stops + at the element that would take it past this, before that + element's members are copied, and fails with ``too_long_message``. + too_long_message: The error for a list past ``max_len``. """ target_type: str convert: Callable[[Any], Any] + max_len: int + too_long_message: str @dataclass(frozen=True, eq=False, **dataclass_kwargs) diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 415b080a..3ee27326 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -1451,14 +1451,26 @@ def _convert_string_range_list_item(value: Any) -> Any: # list item, with target type `T? | list[T]` for element type T. A FLOAT # element's target also accepts int, so an integer keeps the form it is # written in, as in openjd-rs. Without EXPR every element resolves to a string. +# Resolution stops at the element that takes a list past the §3.4 maximum, so +# a template cannot make job creation build a long list just to reject it. +_RANGE_LIST_TOO_LONG = f"range exceeds {_MAX_TASK_PARAM_RANGE_LEN} elements." _INT_RANGE_LIST_ITEM = ListItemSpec( - target_type="int | list[int] | nulltype", convert=_convert_int_range_list_item + target_type="int | list[int] | nulltype", + convert=_convert_int_range_list_item, + max_len=_MAX_TASK_PARAM_RANGE_LEN, + too_long_message=_RANGE_LIST_TOO_LONG, ) _FLOAT_RANGE_LIST_ITEM = ListItemSpec( - target_type="float | int | list[float] | nulltype", convert=_convert_float_range_list_item + target_type="float | int | list[float] | nulltype", + convert=_convert_float_range_list_item, + max_len=_MAX_TASK_PARAM_RANGE_LEN, + too_long_message=_RANGE_LIST_TOO_LONG, ) _STRING_RANGE_LIST_ITEM = ListItemSpec( - target_type="string | list[string] | nulltype", convert=_convert_string_range_list_item + target_type="string | list[string] | nulltype", + convert=_convert_string_range_list_item, + max_len=_MAX_TASK_PARAM_RANGE_LEN, + too_long_message=_RANGE_LIST_TOO_LONG, ) diff --git a/test/openjd/model_v0/test_range_list_items.py b/test/openjd/model_v0/test_range_list_items.py index 55286848..f90fa79a 100644 --- a/test/openjd/model_v0/test_range_list_items.py +++ b/test/openjd/model_v0/test_range_list_items.py @@ -225,6 +225,14 @@ def test_string_and_path_null_drops_and_list_flattens(self, tmp_path, param_type # Text around the expression, even white space, makes the element a # string rather than a list item. assert result == ["/c", "/a", "/b", " x "] + result = _range( + tmp_path, + {"type": param_type, "range": ["{{ 'x' }} ", " {{ 'y' }}"]}, + [{"name": "L", "type": "LIST[STRING]"}], + {"L": "[]"}, + ["EXPR"], + ) + assert result == ["x ", " y"] @pytest.mark.parametrize( "task_param, extensions, target", @@ -306,13 +314,59 @@ def test_list_emptied_by_null_rejected(self, tmp_path, task_param, extensions) - "has no elements after resolution." ) - def test_flattened_list_past_element_limit_rejected(self, tmp_path) -> None: + @pytest.mark.parametrize( + "task_param, extensions, member, scalar", + [ + pytest.param({"type": "INT"}, ["EXPR"], "1", "7", id="INT"), + pytest.param( + {"type": "CHUNK[INT]", "chunks": _CHUNKS}, + ["EXPR", "TASK_CHUNKING"], + "1", + "7", + id="CHUNK", + ), + pytest.param({"type": "FLOAT"}, ["EXPR"], "1.5", "2.5", id="FLOAT"), + pytest.param({"type": "STRING"}, ["EXPR"], "'a'", "b", id="STRING"), + pytest.param({"type": "PATH"}, ["EXPR"], "'a'", "b", id="PATH"), + ], + ) + def test_flattened_list_past_element_limit_rejected( + self, tmp_path, task_param, extensions, member, scalar + ) -> None: + full = f"{{{{ [{member}] * 1024 }}}}" + assert len( + _range(tmp_path, {**task_param, "range": [full]}, _STRING_S, {"S": "x"}, extensions) + ) == (1024) + for range_ in ( + [scalar, full], + [full, scalar], + # Resolution stops at the second element, before its members are + # copied; a ten-million-member list would otherwise take seconds. + [full, "{{ range_expr('1-9999999') }}" if member == "1" else full], + ): + error = _range_error( + tmp_path, {**task_param, "range": range_}, _STRING_S, {"S": "x"}, extensions + ) + assert error == ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range:\n\t" + "range exceeds 1024 elements." + ), range_ + + def test_errors_name_the_element_as_written(self, tmp_path) -> None: + # The element that fails is element 2 of the list as written, even + # though the first element flattened into three values. error = _range_error( tmp_path, - {"type": "INT", "range": [0, "{{ range_expr('1-1024') }}"]}, + {"type": "INT", "range": ["{{ [1, 2, 3] }}", "{{ null }}", "{{ Param.S }}"]}, _STRING_S, {"S": "x"}, ["EXPR"], ) - assert "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range:\n\t" in error - assert "1024" in error + assert error == ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[2]:\n\t" + "Failed to parse interpolation expression at [0, 13]. Expression: Param.S . " + "Reason: Expression failed validation: Cannot coerce string to int | list[int] | nulltype\n" + " Param.S\n ~~~~~~^." + ) diff --git a/test/openjd/model_v1/test_known_gaps.py b/test/openjd/model_v1/test_known_gaps.py index a40e7a13..60129c5c 100644 --- a/test/openjd/model_v1/test_known_gaps.py +++ b/test/openjd/model_v1/test_known_gaps.py @@ -172,3 +172,45 @@ def test_v1_accepts_format_string_in_int_range_list() -> None: ], } ) + + +# With EXPR, v0 resolves a FLOAT range list element as a list item (Expression +# Language §1.3.2), so a list flattens. v1's openjd-model 0.13.0 resolves it to +# text and fails to parse it. Same resolution as the test above. + + +@pytest.mark.xfail(strict=True, reason="bindings pin openjd-model 0.13.0 (FLOAT elements as text)") +def test_v1_flattens_list_in_float_range_list() -> None: + from pathlib import Path + + import openjd.model._v1 as v1 + + template = v1.decode_job_template( + { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "X", "type": "FLOAT", "range": [1.5, "{{ [2, 2.5] }}"]} + ] + }, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + }, + supported_extensions=["EXPR"], + ) + params = v1.preprocess_job_parameters( + job_template=template, + job_parameter_values={}, + job_template_dir=Path.cwd(), + current_working_dir=Path.cwd(), + ) + job = v1.create_job(job_template=template, job_parameter_values=params) + space = job.steps[0].parameter_space + assert space is not None + assert len(space.task_parameter_definitions["X"].range) == 3 From 7347bbf2df841cb779987d7793b549a60b4bdbd2 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:23:23 +0000 Subject: [PATCH 3/4] fix: Address the second review of range list element resolution The second review returned clean; these are its low findings. - When an element fails before a later element takes the list past its limit, report the element error, as openjd-rs does, instead of the limit. - Test FormatString.resolve_list_item's room directly, so stopping before a flattened element's members are copied is pinned rather than only observable as time. - resolve_list_item: type path_format as Optional[Any] like its siblings, and document the None return. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- .../model/_format_strings/_format_string.py | 6 ++- src/openjd/model/_internal/_create_job.py | 5 ++- test/openjd/model_v0/test_range_list_items.py | 39 +++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index ad999616..1b82a3a8 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -241,7 +241,7 @@ def resolve_list_item( symtab: SymbolTable, target_type: str, room: Optional[int] = None, - path_format: Optional[object] = None, + path_format: Optional[Any] = None, ) -> Optional[list[Any]]: """Resolve this format string as a list item (Expression Language §1.3.2). @@ -259,6 +259,10 @@ def resolve_list_item( value references that cannot produce ``null`` or a list) contributes its resolved string, as :meth:`resolve` returns it. + Returns: + The values the element contributes, or ``None`` when it is a list + longer than ``room``. + Raises: FormatStringError: if the expression cannot be evaluated or its result does not coerce to ``target_type``. diff --git a/src/openjd/model/_internal/_create_job.py b/src/openjd/model/_internal/_create_job.py index bce6b514..8b004469 100644 --- a/src/openjd/model/_internal/_create_job.py +++ b/src/openjd/model/_internal/_create_job.py @@ -411,8 +411,11 @@ def _instantiate_list_items( if resolved is not None: result.extend(spec.convert(v) for v in resolved) # The length is a property of the whole list, so it is reported at the - # field rather than at the element that went over. + # field rather than at the element that went over. An element error + # found before it is reported instead, as openjd-rs does. if resolved is None or len(result) > spec.max_len: + if errors: + break raise ValueError(spec.too_long_message) if errors: diff --git a/test/openjd/model_v0/test_range_list_items.py b/test/openjd/model_v0/test_range_list_items.py index f90fa79a..b9125241 100644 --- a/test/openjd/model_v0/test_range_list_items.py +++ b/test/openjd/model_v0/test_range_list_items.py @@ -353,6 +353,22 @@ def test_flattened_list_past_element_limit_rejected( "range exceeds 1024 elements." ), range_ + def test_element_error_before_the_limit_is_reported(self, tmp_path) -> None: + error = _range_error( + tmp_path, + {"type": "INT", "range": ["{{ Param.S }}", "{{ [1] * 1024 }}", 2]}, + _STRING_S, + {"S": "x"}, + ["EXPR"], + ) + assert error == ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[0]:\n\t" + "Failed to parse interpolation expression at [0, 13]. Expression: Param.S . " + "Reason: Expression failed validation: Cannot coerce string to int | list[int] | nulltype\n" + " Param.S\n ~~~~~~^." + ) + def test_errors_name_the_element_as_written(self, tmp_path) -> None: # The element that fails is element 2 of the list as written, even # though the first element flattened into three values. @@ -370,3 +386,26 @@ def test_errors_name_the_element_as_written(self, tmp_path) -> None: "Reason: Expression failed validation: Cannot coerce string to int | list[int] | nulltype\n" " Param.S\n ~~~~~~^." ) + + +class TestResolveListItem: + """``FormatString.resolve_list_item`` directly: a list longer than the + room left is reported before its members are copied.""" + + def test_list_longer_than_room_returns_none(self) -> None: + from openjd.model import SymbolTable + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import ModelParsingContext + + fs = FormatString( + "{{ range_expr('1-9999999') }}", + context=ModelParsingContext(supported_extensions=["EXPR"]), + ) + target = "int | list[int] | nulltype" + assert fs.resolve_list_item(symtab=SymbolTable(), target_type=target, room=1024) is None + small = FormatString( + "{{ [1, 2, 3] }}", context=ModelParsingContext(supported_extensions=["EXPR"]) + ) + assert small.resolve_list_item(symtab=SymbolTable(), target_type=target, room=2) is None + values = small.resolve_list_item(symtab=SymbolTable(), target_type=target, room=3) + assert values is not None and [v.item() for v in values] == [1, 2, 3] From 343a8eff17ed8c9f54944ea5f6daa60e30f8a4c0 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:57:25 +0000 Subject: [PATCH 4/4] test: Update the range list element tests for openjd-model 0.16 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebasing onto the bindings bump to openjd-expr 0.15, openjd-model 0.16 and openjd-sessions 0.10 changed three of these tests' outcomes: - Expression Language §1.2.3 now has string-to-number coercion ignore white space around the number (Template Schemas §7.5), and §1.3.12 gives a STRING value " 7 " as an INT range element as the example. openjd-expr 0.15 does this, so with EXPR v0 now resolves such an element as it does without EXPR. The test that pinned the old error now pins 7 for INT and "2.50" for FLOAT, and new cases pin that white space inside the number, "1 000", is still an error, with and without EXPR. - openjd.model._v1 now accepts a format string in an INT range list and flattens a list in a FLOAT one, so both strict xfails in test_known_gaps.py passed. The INT case is already covered by TestIntRangeListFormatStringElements in test_create_job.py, which the bump added, so it is removed. The FLOAT case moves to TestRangeListElementsAsListItems there, and asserts the values. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- test/openjd/model_v0/test_range_list_items.py | 25 ++++-- test/openjd/model_v1/test_create_job.py | 33 ++++++++ test/openjd/model_v1/test_known_gaps.py | 77 ------------------- 3 files changed, 53 insertions(+), 82 deletions(-) diff --git a/test/openjd/model_v0/test_range_list_items.py b/test/openjd/model_v0/test_range_list_items.py index b9125241..87f32396 100644 --- a/test/openjd/model_v0/test_range_list_items.py +++ b/test/openjd/model_v0/test_range_list_items.py @@ -105,7 +105,7 @@ def test_int_elements_resolve(self, tmp_path, task_param, extensions) -> None: pytest.param({"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["TASK_CHUNKING"], id="CHUNK"), ], ) - @pytest.mark.parametrize("value", ["abc", "2.5", ""]) + @pytest.mark.parametrize("value", ["abc", "2.5", "", "1 000"]) def test_int_element_not_an_integer(self, tmp_path, task_param, extensions, value) -> None: error = _range_error( tmp_path, @@ -263,14 +263,29 @@ def test_list_of_wrong_type_rejected(self, tmp_path, task_param, extensions, tar " Param.L\n ~~~~~~^." ) - def test_padded_string_does_not_coerce(self, tmp_path) -> None: - # Expression Language §1.2.3 coercion does not trim, unlike the - # parse without EXPR. + def test_white_space_around_the_number_is_ignored(self, tmp_path) -> None: + # Expression Language §1.2.3 and §1.3.12: string-to-number coercion + # ignores white space around the number (Template Schemas §7.5), so an + # element resolves as it does without EXPR (test_int_elements_resolve, + # test_float_elements_resolve_and_keep_their_spelling). The white space + # is not part of a float's spelling (§1.3.4). + params = [{"name": "S", "type": "STRING"}, {"name": "T", "type": "STRING"}] + values = {"S": " 7\n", "T": "\t2.50 "} + int_range = _range( + tmp_path, {"type": "INT", "range": [1, "{{Param.S}}"]}, params, values, ["EXPR"] + ) + float_range = _range( + tmp_path, {"type": "FLOAT", "range": [1.5, "{{Param.T}}"]}, params, values, ["EXPR"] + ) + assert int_range == [1, 7] + assert float_range == [Decimal("1.5"), "2.50"] + + def test_white_space_inside_the_number_does_not_coerce(self, tmp_path) -> None: error = _range_error( tmp_path, {"type": "INT", "range": [1, "{{Param.S}}"]}, _STRING_S, - {"S": " 7 "}, + {"S": "1 000"}, ["EXPR"], ) assert error == _ELEMENT_1 + ( diff --git a/test/openjd/model_v1/test_create_job.py b/test/openjd/model_v1/test_create_job.py index 4e825457..1c433efb 100644 --- a/test/openjd/model_v1/test_create_job.py +++ b/test/openjd/model_v1/test_create_job.py @@ -3019,6 +3019,39 @@ def test_a_value_that_is_not_an_integer_fails(self, value: str) -> None: ) +class TestRangeListElementsAsListItems: + """With EXPR, a range list element that is exactly one expression is a list + item (Expression Language §1.3.2): a list it evaluates to flattens into one + element per member. Since openjd-model 0.15.0 (openjd-rs#486); before, a + FLOAT element resolved to text and failed to parse. + ``test_range_list_items.py`` in model_v0 covers v0.""" + + def test_float_list_flattens(self) -> None: + decoded = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "X", "type": "FLOAT", "range": [1.5, "{{ [2, 2.5] }}"]} + ] + }, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + }, + supported_extensions=["EXPR"], + ) + job = create_job(job_template=decoded, job_parameter_values={}) + space = job.steps[0].parameterSpace + assert space is not None + assert space.taskParameterDefinitions["X"].range == [1.5, 2.0, 2.5] + + def _preprocess(parameter_definitions: list[dict], values: dict, tmp_path: Path) -> dict: """Decode a one-step template with ``parameter_definitions`` (with EXPR, for the list types) and return ``preprocess_job_parameters``' values as text.""" diff --git a/test/openjd/model_v1/test_known_gaps.py b/test/openjd/model_v1/test_known_gaps.py index 60129c5c..9966817d 100644 --- a/test/openjd/model_v1/test_known_gaps.py +++ b/test/openjd/model_v1/test_known_gaps.py @@ -137,80 +137,3 @@ def test_v0_range_string_renders_in_canonical_form(text: str, expected: str) -> from openjd.model._range_expr import IntRangeExpr assert str(IntRangeExpr.from_str(text)) == expected - - -# ── v1 rejects format string elements in INT range lists ── -# -# Template Schemas §3.4.1.1 allows a element in an -# INT (and CHUNK[INT]) range list, and v0 accepts one. The bindings build -# against openjd-model 0.13.0, whose `IntRange::List` holds integers only, so v1 -# fails to decode the template. The openjd-rs fix changes `IntRange::List` to -# `Vec`; this resolves when the bindings move to an -# openjd-model release with it, and `int_range_to_py` handles the new element -# type. ``test_range_list_items.py`` in model_v0 covers the v0 behavior. - - -@pytest.mark.xfail(strict=True, reason="bindings pin openjd-model 0.13.0 (IntRange::List of i64)") -def test_v1_accepts_format_string_in_int_range_list() -> None: - import openjd.model._v1 as v1 - - v1.decode_job_template( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "T", - "parameterDefinitions": [{"name": "I", "type": "INT", "default": 10}], - "steps": [ - { - "name": "S", - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "X", "type": "INT", "range": [1, "{{Param.I}}"]} - ] - }, - "script": {"actions": {"onRun": {"command": "echo"}}}, - } - ], - } - ) - - -# With EXPR, v0 resolves a FLOAT range list element as a list item (Expression -# Language §1.3.2), so a list flattens. v1's openjd-model 0.13.0 resolves it to -# text and fails to parse it. Same resolution as the test above. - - -@pytest.mark.xfail(strict=True, reason="bindings pin openjd-model 0.13.0 (FLOAT elements as text)") -def test_v1_flattens_list_in_float_range_list() -> None: - from pathlib import Path - - import openjd.model._v1 as v1 - - template = v1.decode_job_template( - { - "specificationVersion": "jobtemplate-2023-09", - "extensions": ["EXPR"], - "name": "T", - "steps": [ - { - "name": "S", - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "X", "type": "FLOAT", "range": [1.5, "{{ [2, 2.5] }}"]} - ] - }, - "script": {"actions": {"onRun": {"command": "echo"}}}, - } - ], - }, - supported_extensions=["EXPR"], - ) - params = v1.preprocess_job_parameters( - job_template=template, - job_parameter_values={}, - job_template_dir=Path.cwd(), - current_working_dir=Path.cwd(), - ) - job = v1.create_job(job_template=template, job_parameter_values=params) - space = job.steps[0].parameter_space - assert space is not None - assert len(space.task_parameter_definitions["X"].range) == 3