diff --git a/src/openjd/model/_format_strings/_expression.py b/src/openjd/model/_format_strings/_expression.py index 844dc7c0..81681b15 100644 --- a/src/openjd/model/_format_strings/_expression.py +++ b/src/openjd/model/_format_strings/_expression.py @@ -86,9 +86,24 @@ def evaluate( raise ExpressionError(f"Nonvalid result type: {result} of type {type(result)}") - def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = None) -> Any: + @property + def is_expr(self) -> bool: + """Whether this expression uses the EXPR language (Expression Language), + rather than the base 2023-09 value-reference grammar.""" + return getattr(self._expresion_tree, "allows_nonscalar", False) + + def evaluate_value( + self, + *, + symtab: SymbolTable, + path_format: Optional[Any] = None, + target_type: Optional[Any] = None, + ) -> Any: """Evaluate the expression, preferring the engine's typed value form. + ``target_type`` (an ``openjd.expr.ExprType``), when given, applies to + EXPR-backed expressions: the result is coerced toward it. + For EXPR-backed expressions this returns the engine ``ExprValue``, which keeps the result's EXPR type (path/int/float/list/...) and its string-rendering fidelity when re-seeded into a symbol table — the @@ -103,6 +118,10 @@ def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = No if evaluate_value is None: return self.evaluate(symtab=symtab, path_format=path_format) try: + if target_type is not None: + return evaluate_value( + symtab=symtab, path_format=path_format, target_type=target_type + ) return evaluate_value(symtab=symtab, path_format=path_format) except ValueError as exc: raise ExpressionError(f"Expression failed validation: {str(exc)}") diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index b47245e2..1b82a3a8 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import functools from dataclasses import dataclass from typing import Any, Optional, Union @@ -10,6 +11,16 @@ from .._types import ModelParsingContextInterface, SpecificationRevision +@functools.lru_cache(maxsize=None) +def _expr_type(type_string: str) -> Any: + """The ``openjd.expr.ExprType`` for ``type_string``. Imported here, on + first use, because importing openjd.model must not load the native + extension; only an EXPR template reaches the callers.""" + from openjd.expr import ExprType + + return ExprType(type_string) + + class _ReconstructionContext(ModelParsingContextInterface): """Minimal parsing context used when a FormatString is reconstructed by pickle or the copy module (see ``FormatString.__getnewargs_ex__``). @@ -224,6 +235,67 @@ def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] = ) return self.resolve(symtab=symtab, path_format=path_format) + def resolve_list_item( + self, + *, + symtab: SymbolTable, + target_type: str, + room: Optional[int] = None, + path_format: Optional[Any] = None, + ) -> Optional[list[Any]]: + """Resolve this format string as a list item (Expression Language §1.3.2). + + When the format string is exactly one EXPR ``{{ ... }}`` expression, + with no text around it, the expression is evaluated with + ``target_type`` (an EXPR type string of the form ``T? | list[T]``, + such as ``"int | list[int] | nulltype"``): a ``null`` result contributes no values, a list + contributes one value per member, and any other result contributes + itself. The values are the engine's ``ExprValue``s. A result that does + not coerce to the target raises. A list longer than ``room`` (when + given) returns ``None`` before its members are copied. + + Any other format string (text around its expressions, several + expressions, or a template without EXPR, whose expressions are bare + value references that cannot produce ``null`` or a list) contributes + its resolved string, as :meth:`resolve` returns it. + + Returns: + The values the element contributes, or ``None`` when it is a list + longer than ``room``. + + Raises: + FormatStringError: if the expression cannot be evaluated or its + result does not coerce to ``target_type``. + """ + expressions = self.expressions + if len(expressions) == 1: + info = expressions[0] + expression = info.expression + whole_field = info.start_pos == 0 and info.end_pos == len(self.original_value) + if whole_field and expression is not None and getattr(expression, "is_expr", False): + try: + value = expression.evaluate_value( + symtab=symtab, + path_format=path_format, + target_type=_expr_type(target_type), + ) + except ExpressionError as exc: + raise FormatStringError( + string=self.original_value, + start=info.start_pos, + end=info.end_pos, + expr=expression.expr, + details=str(exc), + ) + if getattr(value, "is_null", False): + return [] + if str(value.type).startswith("list["): + if room is not None and len(value) > room: + return None + return list(value) + return [value] + return [self.resolve(symtab=symtab, path_format=path_format)] + def _preprocess( self, *, context: ModelParsingContextInterface ) -> list[Union[str, ExpressionInfo]]: diff --git a/src/openjd/model/_format_strings/_nodes.py b/src/openjd/model/_format_strings/_nodes.py index 750283b6..5f760efb 100644 --- a/src/openjd/model/_format_strings/_nodes.py +++ b/src/openjd/model/_format_strings/_nodes.py @@ -220,12 +220,17 @@ def _check_comprehension_shadowing(self, symbols: set) -> None: "a variable that already exists at this location." ) - def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: + def _evaluate_raw( + self, *, symtab: SymbolTable, path_format: Any = None, target_type: Any = None + ) -> Any: """Evaluate the expression and return the engine's ``ExprValue``. Shared by :meth:`evaluate` (which unwraps to the native Python value) and :meth:`evaluate_to_str` (which uses the engine's spec-defined string coercion). Re-raises engine errors as the model's ``ExpressionError``. + ``target_type`` (an ``openjd.expr.ExprType``), when given, is the + result's target type (Expression Language §1.2.3): the result is + coerced toward it, and evaluation fails when it cannot be. """ from ._expr_support import ( map_eval_error, @@ -246,16 +251,20 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: # is not absolute, and that reaches the caller as an expression # error like any other. profile = profile_for_symtab(symtab) - return self._parsed.evaluate(values=values, profile=profile, path_format=path_format) + return self._parsed.evaluate( + values=values, profile=profile, path_format=path_format, target_type=target_type + ) except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error raise map_eval_error(exc) def evaluate(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: return self._evaluate_raw(symtab=symtab, path_format=path_format).item() - def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any: + def evaluate_value( + self, *, symtab: SymbolTable, path_format: Any = None, target_type: Any = None + ) -> Any: """Evaluate and return the engine's ``ExprValue`` (not unwrapped to a - native Python value). + native Python value), coerced toward ``target_type`` when given. Callers that re-seed the result into a symbol table (e.g. the session runtime's ``let`` bindings, RFC 0007) should use this form: the typed @@ -265,7 +274,7 @@ def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any are preserved), matching the Rust runtime's natively typed symbol table. """ - return self._evaluate_raw(symtab=symtab, path_format=path_format) + return self._evaluate_raw(symtab=symtab, path_format=path_format, target_type=target_type) def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Any = None) -> str: # ``str(ExprValue)`` applies the engine's RFC 0005 format-string diff --git a/src/openjd/model/_internal/_create_job.py b/src/openjd/model/_internal/_create_job.py index f5b47f3c..8b004469 100644 --- a/src/openjd/model/_internal/_create_job.py +++ b/src/openjd/model/_internal/_create_job.py @@ -9,7 +9,7 @@ from .._symbol_table import SymbolTable from .._format_strings import FormatString -from .._types import ModelParsingContextInterface, OpenJDModel +from .._types import ListItemSpec, ModelParsingContextInterface, OpenJDModel __all__ = ("instantiate_model", "resolve_whole_field_typed_list") @@ -262,6 +262,15 @@ def _instantiate_model( # noqa: C901 instantiated = coerce(model, field_name, raw_typed) else: instantiated = unwrap_typed_resolution(raw_typed) + elif ( + isinstance(field_value, list) + and field_name in model._job_creation_metadata.list_item_fields + ): + instantiated = _instantiate_list_items( + field_value, + symtab, + model._job_creation_metadata.list_item_fields[field_name], + ) elif isinstance(field_value, list): if field_name in model._job_creation_metadata.reshape_field_to_dict: key_field = model._job_creation_metadata.reshape_field_to_dict[field_name] @@ -366,6 +375,60 @@ def _instantiate_list_field_as_list( # noqa: C901 return result +def _instantiate_list_items( + value: list[Any], + symtab: SymbolTable, + spec: ListItemSpec, +) -> list[Any]: + """Instantiate a list whose format string elements are list items + (Expression Language §1.3.2; see ``JobCreationMetadata.list_item_fields``). + + Each format string element contributes the values + :meth:`FormatString.resolve_list_item` resolves it to, and every value, + and every element that is not a format string, goes through + ``spec.convert``. Errors are reported at the element's index in the list + as written. Resolution stops at the first element that takes the list + past ``spec.max_len``, before a flattened element's members are copied. + + Raises: + ValidationError: if an element fails to resolve or convert. + ValueError: if the list resolves to more than ``spec.max_len`` + elements, or a list that was not empty resolves to none. + """ + errors: list[InitErrorDetails] = [] + result: list[Any] = [] + for idx, item in enumerate(value): + resolved: Optional[list[Any]] = [] + with capture_validation_errors(output_errors=errors, loc=(idx,), input=value): + if isinstance(item, FormatString): + resolved = item.resolve_list_item( + symtab=symtab, + target_type=spec.target_type, + room=spec.max_len - len(result), + ) + else: + resolved = [item] + if resolved is not None: + result.extend(spec.convert(v) for v in resolved) + # The length is a property of the whole list, so it is reported at the + # field rather than at the element that went over. An element error + # found before it is reported instead, as openjd-rs does. + if resolved is None or len(result) > spec.max_len: + if errors: + break + raise ValueError(spec.too_long_message) + + if errors: + raise ValidationError.from_exception_data( + title="_instantiate_list_items", line_errors=errors + ) + # The non-empty rule is checked on the list as written, at decode; dropped + # elements can still leave it empty. + if value and not result: + raise ValueError("has no elements after resolution.") + return result + + def _instantiate_list_field_as_dict( # noqa: C901 value: list[Any], symtab: SymbolTable, diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index e4174f66..f0be2d2b 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -258,6 +258,30 @@ class JobCreateAsMetadata: callable: Optional[Callable[["OpenJDModel", dict], Type["OpenJDModel"]]] = field(default=None) +@dataclass(frozen=True, eq=False, **dataclass_kwargs) +class ListItemSpec: + """How the elements of a list field resolve as list items (see + ``JobCreationMetadata.list_item_fields``). + + Attributes: + target_type: The EXPR type string of a whole-field EXPR element's + target, of the form ``T? | list[T]``, such as + ``"int | list[int] | nulltype"``. + convert: Turns one resolved value, an ``openjd.expr.ExprValue`` from a + typed element or the ``str`` of any other element, into the + instantiated element. Raises ``ValueError`` to reject it. + max_len: The most elements the list may resolve to. Resolution stops + at the element that would take it past this, before that + element's members are copied, and fails with ``too_long_message``. + too_long_message: The error for a list past ``max_len``. + """ + + target_type: str + convert: Callable[[Any], Any] + max_len: int + too_long_message: str + + @dataclass(frozen=True, eq=False, **dataclass_kwargs) class JobCreationMetadata: """Metadata for creating a job out of a job template. @@ -302,6 +326,19 @@ class JobCreationMetadata: native Python list unchanged. """ + list_item_fields: dict[str, "ListItemSpec"] = field(default_factory=dict) + """List fields (a subset of ``resolve_fields``) whose format string + elements are list items (Expression Language §1.3.2), keyed by field name. + Each element resolves through :meth:`FormatString.resolve_list_item` with + the spec's target type, so with EXPR a whole-field ``null`` drops the + element and a list flattens into one element per member. The spec's + ``convert`` turns each resolved value into the instantiated element, and + may raise ``ValueError`` to reject it; the error is reported at the + element's index in the list as written. A list that was not empty as + written but resolves to no elements is rejected. Used by task-parameter + ``range`` lists. + """ + create_as: Optional[JobCreateAsMetadata] = field(default=None) """The class to create the model as during instantiation. Options: 1. None -- create as the model class itself. diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 13a3a286..3ee27326 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import re import secrets import string @@ -55,6 +56,7 @@ JobCreateAsMetadata, JobCreationMetadata, JobParameterInterface, + ListItemSpec, ModelParsingContextInterface, OpenJDModel, ResolutionScope, @@ -1399,6 +1401,79 @@ def _range_element_type_name(elem: Any) -> str: return "list" if type_str.startswith("list[") else type_str +def _expr_value_type(value: Any) -> Optional[str]: + """The EXPR type of an engine ``ExprValue``, or ``None`` for a native value.""" + value_type = getattr(value, "type", None) + return None if value_type is None else str(value_type) + + +def _convert_int_range_list_item(value: Any) -> int: + """One INT or CHUNK[INT] range list element (Template Schemas §3.4.1.1), + from a literal, a typed list item, or a resolved format string, which + must be the string representation of an integer.""" + if _expr_value_type(value) == "int": + return value.item() + if isinstance(value, str): + try: + return int(value) + except ValueError: + raise ValueError(f"value {value!r} is not an integer.") + return value + + +def _convert_float_range_list_item(value: Any) -> Any: + """One FLOAT range list element (Template Schemas §3.4.1.2). A typed list + item becomes its string form, which keeps a float's spelling and writes an + integer as written (openjd-rs does the same), and a resolved string must + be the string representation of a finite floating point value. Strings + are normalized by ``RangeListTaskParameterDefinition`` (§7.5).""" + if _expr_value_type(value) is not None: + value = str(value) + if isinstance(value, str): + try: + number = float(value) + except ValueError: + raise ValueError(f"value {value!r} is not a number.") + if not math.isfinite(number): + raise ValueError(f"value {value!r} is not finite.") + return value + + +def _convert_string_range_list_item(value: Any) -> Any: + """One STRING or PATH range list element: a typed list item becomes its + string form (Expression Language string coercion).""" + if _expr_value_type(value) is not None: + return str(value) + return value + + +# Expression Language §1.3.2: each element of a task parameter range list is a +# list item, with target type `T? | list[T]` for element type T. A FLOAT +# element's target also accepts int, so an integer keeps the form it is +# written in, as in openjd-rs. Without EXPR every element resolves to a string. +# Resolution stops at the element that takes a list past the §3.4 maximum, so +# a template cannot make job creation build a long list just to reject it. +_RANGE_LIST_TOO_LONG = f"range exceeds {_MAX_TASK_PARAM_RANGE_LEN} elements." +_INT_RANGE_LIST_ITEM = ListItemSpec( + target_type="int | list[int] | nulltype", + convert=_convert_int_range_list_item, + max_len=_MAX_TASK_PARAM_RANGE_LEN, + too_long_message=_RANGE_LIST_TOO_LONG, +) +_FLOAT_RANGE_LIST_ITEM = ListItemSpec( + target_type="float | int | list[float] | nulltype", + convert=_convert_float_range_list_item, + max_len=_MAX_TASK_PARAM_RANGE_LEN, + too_long_message=_RANGE_LIST_TOO_LONG, +) +_STRING_RANGE_LIST_ITEM = ListItemSpec( + target_type="string | list[string] | nulltype", + convert=_convert_string_range_list_item, + max_len=_MAX_TASK_PARAM_RANGE_LEN, + too_long_message=_RANGE_LIST_TOO_LONG, +) + + def _coerce_typed_range_elements(model: Any, field_name: str, raw: Any) -> list: """``typed_resolve_coerce`` hook for task-parameter ``range`` fields. @@ -1521,6 +1596,7 @@ class IntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_09 resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _INT_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) @@ -1589,6 +1665,7 @@ class FloatTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_ resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _FLOAT_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) @@ -1635,6 +1712,7 @@ class StringTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023 resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _STRING_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) @@ -1670,6 +1748,7 @@ class PathTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v2023_0 resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _STRING_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) @@ -1725,6 +1804,7 @@ class ChunkIntTaskParameterDefinition(NameIdentifierLengthMixin, OpenJDModel_v20 resolve_fields={"range"}, typed_resolve_fields={"range"}, typed_resolve_coerce=_coerce_typed_range_elements, + list_item_fields={"range": _INT_RANGE_LIST_ITEM}, exclude_fields={"name"}, ) diff --git a/test/openjd/model_v0/test_range_list_items.py b/test/openjd/model_v0/test_range_list_items.py new file mode 100644 index 00000000..87f32396 --- /dev/null +++ b/test/openjd/model_v0/test_range_list_items.py @@ -0,0 +1,426 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""Job creation for the format string elements of task parameter range lists. + +Template Schemas §3.4.1.1 and §3.4.1.2: an element of an INT, CHUNK[INT] or +FLOAT range list may be a , which must resolve to +the string representation of a number. With the EXPR extension, Expression +Language §1.3.2 makes every range list element a list item, with target type +``T? | list[T]``: a whole-field ``null`` drops the element, a list flattens +into one element per member, and a value that does not coerce is rejected. +openjd-rs (crates/openjd-model/src/job/create_job/ranges.rs) does the same. +""" + +from decimal import Decimal +from pathlib import Path +from typing import Any, Optional + +import pytest + +from openjd.model import ( + DecodeValidationError, + create_job, + decode_job_template, + preprocess_job_parameters, +) + + +def _range( + tmp_path: Path, + task_param: dict[str, Any], + params: list[dict[str, Any]], + values: dict[str, str], + extensions: Optional[list[str]] = None, +) -> list[Any]: + """Create a job with one task parameter, X, and return its resolved range.""" + template: dict[str, Any] = { + "specificationVersion": "jobtemplate-2023-09", + "name": "T", + "parameterDefinitions": params, + "steps": [ + { + "name": "S", + "parameterSpace": {"taskParameterDefinitions": [{"name": "X", **task_param}]}, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + } + if extensions: + template["extensions"] = extensions + job_template = decode_job_template( + template=template, supported_extensions=["EXPR", "TASK_CHUNKING"] + ) + processed = preprocess_job_parameters( + job_template=job_template, + job_parameter_values=values, + job_template_dir=tmp_path, + current_working_dir=tmp_path, + ) + job = create_job(job_template=job_template, job_parameter_values=processed) + assert job.steps[0].parameterSpace is not None + return list(job.steps[0].parameterSpace.taskParameterDefinitions["X"].range) + + +def _range_error(tmp_path: Path, *args: Any, **kwargs: Any) -> str: + with pytest.raises(DecodeValidationError) as excinfo: + _range(tmp_path, *args, **kwargs) + return str(excinfo.value) + + +_CHUNKS = {"defaultTaskCount": 1, "rangeConstraint": "NONCONTIGUOUS"} +_STRING_S = [{"name": "S", "type": "STRING"}] +_ELEMENT_1 = ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[1]:\n\t" +) + + +class TestWithoutExpr: + """Without EXPR an element resolves to a string, parsed as an + or .""" + + @pytest.mark.parametrize( + "task_param, extensions", + [ + pytest.param({"type": "INT"}, None, id="INT"), + pytest.param({"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["TASK_CHUNKING"], id="CHUNK"), + ], + ) + def test_int_elements_resolve(self, tmp_path, task_param, extensions) -> None: + params = [{"name": "I", "type": "INT"}, {"name": "S", "type": "STRING"}] + result = _range( + tmp_path, + {**task_param, "range": [1, "2", "{{Param.I}}", "{{Param.S}}", "1{{Param.I}}"]}, + params, + {"I": "7", "S": " 9 "}, + extensions, + ) + assert result == [1, 2, 7, 9, 17] + + @pytest.mark.parametrize( + "task_param, extensions", + [ + pytest.param({"type": "INT"}, None, id="INT"), + # CHUNK[INT] used to keep the text and raise an uncaught ValueError + # when the parameter space was iterated. + pytest.param({"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["TASK_CHUNKING"], id="CHUNK"), + ], + ) + @pytest.mark.parametrize("value", ["abc", "2.5", "", "1 000"]) + def test_int_element_not_an_integer(self, tmp_path, task_param, extensions, value) -> None: + error = _range_error( + tmp_path, + {**task_param, "range": [1, "{{Param.S}}"]}, + _STRING_S, + {"S": value}, + extensions, + ) + assert error == _ELEMENT_1 + f"value {value!r} is not an integer." + + def test_float_elements_resolve_and_keep_their_spelling(self, tmp_path) -> None: + params = [{"name": "F", "type": "FLOAT"}, {"name": "I", "type": "INT"}] + result = _range( + tmp_path, + {"type": "FLOAT", "range": [1.5, "{{Param.F}}", "{{Param.I}}", " 02.50 "]}, + params, + {"F": "2.50", "I": "10"}, + ) + assert result == [Decimal("1.5"), "2.50", "10", "2.50"] + + @pytest.mark.parametrize( + "value, problem", + [ + ("abc", "is not a number"), + ("", "is not a number"), + ("inf", "is not finite"), + ("nan", "is not finite"), + ], + ) + def test_float_element_not_a_finite_number(self, tmp_path, value, problem) -> None: + error = _range_error( + tmp_path, {"type": "FLOAT", "range": [1.5, "{{Param.S}}"]}, _STRING_S, {"S": value} + ) + assert error == _ELEMENT_1 + f"value {value!r} {problem}." + + def test_string_elements_are_not_list_items(self, tmp_path) -> None: + # Without EXPR an element cannot produce null or a list, and resolves + # to its string unchanged. + result = _range( + tmp_path, {"type": "STRING", "range": ["a", " {{Param.S}} "]}, _STRING_S, {"S": "b"} + ) + assert result == ["a", " b "] + + +class TestWithExpr: + """With EXPR a whole-field element is a list item (Expression Language §1.3.2).""" + + @pytest.mark.parametrize( + "task_param, extensions", + [ + pytest.param({"type": "INT"}, ["EXPR"], id="INT"), + pytest.param( + {"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["EXPR", "TASK_CHUNKING"], id="CHUNK" + ), + ], + ) + def test_int_null_drops_list_flattens_and_coerces( + self, tmp_path, task_param, extensions + ) -> None: + params = [{"name": "L", "type": "LIST[INT]"}, {"name": "S", "type": "STRING"}] + result = _range( + tmp_path, + { + **task_param, + "range": [ + 1, + "{{ null }}", + "{{ Param.L }}", + "{{ range_expr('7-8') }}", + "{{ 3.0 * 3 }}", + "{{ Param.S }}", + ], + }, + params, + {"L": "[3, 4]", "S": "5"}, + extensions, + ) + assert result == [1, 3, 4, 7, 8, 9, 5] + + def test_float_null_drops_list_flattens_and_keeps_spelling(self, tmp_path) -> None: + params = [ + {"name": "F", "type": "FLOAT"}, + {"name": "I", "type": "INT"}, + {"name": "L", "type": "LIST[INT]"}, + ] + result = _range( + tmp_path, + { + "type": "FLOAT", + "range": [ + 1.5, + "{{ null }}", + "{{ Param.F }}", + "{{ Param.I }}", + "{{ [Param.F] }}", + "{{ Param.L }}", + ], + }, + params, + {"F": "2.50", "I": "10", "L": "[3, 4]"}, + ["EXPR"], + ) + # A float keeps its spelling, an integer is written as is, and the + # integer members of a list converted to float are written as floats, + # as in openjd-rs. + assert result == [Decimal("1.5"), "2.50", "10", "2.50", "3.0", "4.0"] + + @pytest.mark.parametrize("param_type", ["STRING", "PATH"]) + def test_string_and_path_null_drops_and_list_flattens(self, tmp_path, param_type) -> None: + result = _range( + tmp_path, + {"type": param_type, "range": ["/c", "{{ null }}", "{{ Param.L }}", " {{ 'x' }} "]}, + [{"name": "L", "type": "LIST[STRING]"}], + {"L": '["/a", "/b"]'}, + ["EXPR"], + ) + # Text around the expression, even white space, makes the element a + # string rather than a list item. + assert result == ["/c", "/a", "/b", " x "] + result = _range( + tmp_path, + {"type": param_type, "range": ["{{ 'x' }} ", " {{ 'y' }}"]}, + [{"name": "L", "type": "LIST[STRING]"}], + {"L": "[]"}, + ["EXPR"], + ) + assert result == ["x ", " y"] + + @pytest.mark.parametrize( + "task_param, extensions, target", + [ + pytest.param({"type": "INT"}, ["EXPR"], "int | list[int] | nulltype", id="INT"), + pytest.param( + {"type": "CHUNK[INT]", "chunks": _CHUNKS}, + ["EXPR", "TASK_CHUNKING"], + "int | list[int] | nulltype", + id="CHUNK", + ), + pytest.param( + {"type": "FLOAT"}, ["EXPR"], "float | int | list[float] | nulltype", id="FLOAT" + ), + ], + ) + def test_list_of_wrong_type_rejected(self, tmp_path, task_param, extensions, target) -> None: + error = _range_error( + tmp_path, + {**task_param, "range": [1, "{{ Param.L }}"]}, + [{"name": "L", "type": "LIST[STRING]"}], + {"L": '["a"]'}, + extensions, + ) + assert error == _ELEMENT_1 + ( + "Failed to parse interpolation expression at [0, 13]. Expression: Param.L . " + f"Reason: Expression failed validation: Cannot coerce list[string] to {target}\n" + " Param.L\n ~~~~~~^." + ) + + def test_white_space_around_the_number_is_ignored(self, tmp_path) -> None: + # Expression Language §1.2.3 and §1.3.12: string-to-number coercion + # ignores white space around the number (Template Schemas §7.5), so an + # element resolves as it does without EXPR (test_int_elements_resolve, + # test_float_elements_resolve_and_keep_their_spelling). The white space + # is not part of a float's spelling (§1.3.4). + params = [{"name": "S", "type": "STRING"}, {"name": "T", "type": "STRING"}] + values = {"S": " 7\n", "T": "\t2.50 "} + int_range = _range( + tmp_path, {"type": "INT", "range": [1, "{{Param.S}}"]}, params, values, ["EXPR"] + ) + float_range = _range( + tmp_path, {"type": "FLOAT", "range": [1.5, "{{Param.T}}"]}, params, values, ["EXPR"] + ) + assert int_range == [1, 7] + assert float_range == [Decimal("1.5"), "2.50"] + + def test_white_space_inside_the_number_does_not_coerce(self, tmp_path) -> None: + error = _range_error( + tmp_path, + {"type": "INT", "range": [1, "{{Param.S}}"]}, + _STRING_S, + {"S": "1 000"}, + ["EXPR"], + ) + assert error == _ELEMENT_1 + ( + "Failed to parse interpolation expression at [0, 11]. Expression: Param.S. " + "Reason: Expression failed validation: Cannot coerce string to int | list[int] | nulltype\n" + " Param.S\n ~~~~~~^." + ) + + def test_text_around_expression_resolves_to_string(self, tmp_path) -> None: + result = _range( + tmp_path, + {"type": "INT", "range": ["1{{ Param.S }}", "{{ 2 }}{{ 3 }}"]}, + _STRING_S, + {"S": "5"}, + ["EXPR"], + ) + assert result == [15, 23] + + @pytest.mark.parametrize( + "task_param, extensions", + [ + pytest.param({"type": "INT"}, ["EXPR"], id="INT"), + pytest.param({"type": "FLOAT"}, ["EXPR"], id="FLOAT"), + pytest.param({"type": "STRING"}, ["EXPR"], id="STRING"), + pytest.param( + {"type": "CHUNK[INT]", "chunks": _CHUNKS}, ["EXPR", "TASK_CHUNKING"], id="CHUNK" + ), + ], + ) + def test_list_emptied_by_null_rejected(self, tmp_path, task_param, extensions) -> None: + error = _range_error( + tmp_path, + {**task_param, "range": ["{{ null }}", "{{ null if Param.S == 'x' else 1 }}"]}, + _STRING_S, + {"S": "x"}, + extensions, + ) + assert error == ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range:\n\t" + "has no elements after resolution." + ) + + @pytest.mark.parametrize( + "task_param, extensions, member, scalar", + [ + pytest.param({"type": "INT"}, ["EXPR"], "1", "7", id="INT"), + pytest.param( + {"type": "CHUNK[INT]", "chunks": _CHUNKS}, + ["EXPR", "TASK_CHUNKING"], + "1", + "7", + id="CHUNK", + ), + pytest.param({"type": "FLOAT"}, ["EXPR"], "1.5", "2.5", id="FLOAT"), + pytest.param({"type": "STRING"}, ["EXPR"], "'a'", "b", id="STRING"), + pytest.param({"type": "PATH"}, ["EXPR"], "'a'", "b", id="PATH"), + ], + ) + def test_flattened_list_past_element_limit_rejected( + self, tmp_path, task_param, extensions, member, scalar + ) -> None: + full = f"{{{{ [{member}] * 1024 }}}}" + assert len( + _range(tmp_path, {**task_param, "range": [full]}, _STRING_S, {"S": "x"}, extensions) + ) == (1024) + for range_ in ( + [scalar, full], + [full, scalar], + # Resolution stops at the second element, before its members are + # copied; a ten-million-member list would otherwise take seconds. + [full, "{{ range_expr('1-9999999') }}" if member == "1" else full], + ): + error = _range_error( + tmp_path, {**task_param, "range": range_}, _STRING_S, {"S": "x"}, extensions + ) + assert error == ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range:\n\t" + "range exceeds 1024 elements." + ), range_ + + def test_element_error_before_the_limit_is_reported(self, tmp_path) -> None: + error = _range_error( + tmp_path, + {"type": "INT", "range": ["{{ Param.S }}", "{{ [1] * 1024 }}", 2]}, + _STRING_S, + {"S": "x"}, + ["EXPR"], + ) + assert error == ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[0]:\n\t" + "Failed to parse interpolation expression at [0, 13]. Expression: Param.S . " + "Reason: Expression failed validation: Cannot coerce string to int | list[int] | nulltype\n" + " Param.S\n ~~~~~~^." + ) + + def test_errors_name_the_element_as_written(self, tmp_path) -> None: + # The element that fails is element 2 of the list as written, even + # though the first element flattened into three values. + error = _range_error( + tmp_path, + {"type": "INT", "range": ["{{ [1, 2, 3] }}", "{{ null }}", "{{ Param.S }}"]}, + _STRING_S, + {"S": "x"}, + ["EXPR"], + ) + assert error == ( + "1 validation errors for JobTemplate\n" + "steps[0] -> parameterSpace -> taskParameterDefinitions[0] -> range[2]:\n\t" + "Failed to parse interpolation expression at [0, 13]. Expression: Param.S . " + "Reason: Expression failed validation: Cannot coerce string to int | list[int] | nulltype\n" + " Param.S\n ~~~~~~^." + ) + + +class TestResolveListItem: + """``FormatString.resolve_list_item`` directly: a list longer than the + room left is reported before its members are copied.""" + + def test_list_longer_than_room_returns_none(self) -> None: + from openjd.model import SymbolTable + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import ModelParsingContext + + fs = FormatString( + "{{ range_expr('1-9999999') }}", + context=ModelParsingContext(supported_extensions=["EXPR"]), + ) + target = "int | list[int] | nulltype" + assert fs.resolve_list_item(symtab=SymbolTable(), target_type=target, room=1024) is None + small = FormatString( + "{{ [1, 2, 3] }}", context=ModelParsingContext(supported_extensions=["EXPR"]) + ) + assert small.resolve_list_item(symtab=SymbolTable(), target_type=target, room=2) is None + values = small.resolve_list_item(symtab=SymbolTable(), target_type=target, room=3) + assert values is not None and [v.item() for v in values] == [1, 2, 3] diff --git a/test/openjd/model_v1/test_create_job.py b/test/openjd/model_v1/test_create_job.py index 4e825457..1c433efb 100644 --- a/test/openjd/model_v1/test_create_job.py +++ b/test/openjd/model_v1/test_create_job.py @@ -3019,6 +3019,39 @@ def test_a_value_that_is_not_an_integer_fails(self, value: str) -> None: ) +class TestRangeListElementsAsListItems: + """With EXPR, a range list element that is exactly one expression is a list + item (Expression Language §1.3.2): a list it evaluates to flattens into one + element per member. Since openjd-model 0.15.0 (openjd-rs#486); before, a + FLOAT element resolved to text and failed to parse. + ``test_range_list_items.py`` in model_v0 covers v0.""" + + def test_float_list_flattens(self) -> None: + decoded = decode_job_template( + template={ + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["EXPR"], + "name": "T", + "steps": [ + { + "name": "S", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "X", "type": "FLOAT", "range": [1.5, "{{ [2, 2.5] }}"]} + ] + }, + "script": {"actions": {"onRun": {"command": "echo"}}}, + } + ], + }, + supported_extensions=["EXPR"], + ) + job = create_job(job_template=decoded, job_parameter_values={}) + space = job.steps[0].parameterSpace + assert space is not None + assert space.taskParameterDefinitions["X"].range == [1.5, 2.0, 2.5] + + def _preprocess(parameter_definitions: list[dict], values: dict, tmp_path: Path) -> dict: """Decode a one-step template with ``parameter_definitions`` (with EXPR, for the list types) and return ``preprocess_job_parameters``' values as text."""