Skip to content

Commit 721b3e7

Browse files
committed
fix!: Resolve task parameter range list elements as list items
Template Schemas §3.4.1.1 and §3.4.1.2 require a <TaskParameterStringValue> element of an INT, CHUNK[INT] or FLOAT range list to resolve to a number, but job creation kept any resolved text as the value: an INT parameter could take the value "abc", and a CHUNK[INT] one raised an uncaught ValueError when its parameter space was iterated. Such an element is now rejected at job creation, at range[k], as "value 'abc' is not an integer." (or "is not a number." / "is not finite." for FLOAT). With the EXPR extension, Expression Language §1.3.2 makes every range list element a list item, for all five task parameter types. An element that is exactly one {{ ... }} expression is evaluated with the target type T? | list[T]: null drops the element, a list flattens into one element per member, and a value that does not coerce to the element type is rejected. A range emptied by dropped elements is rejected with "has no elements after resolution." An element with text around its expression still resolves to a string, as does every element without EXPR. This matches openjd-rs. - FormatString.resolve_list_item and JobCreationMetadata.list_item_fields (ListItemSpec) implement the list-item resolution; the expression nodes accept a target type. openjd.expr is still loaded only on use. - test_range_list_items.py covers both modes; test_known_gaps.py records that openjd.model._v1 still rejects INT range format strings until the bindings move to an openjd-model release with the fix. BREAKING CHANGE: with EXPR, a STRING or PATH range element that evaluates to a list now gives one task per member instead of one task whose value is the list's text, and an element that evaluates to null is dropped instead of giving an empty string. An INT, CHUNK[INT] or FLOAT element that resolves to text that is not a number now fails job creation. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com>
1 parent 1eaa3c0 commit 721b3e7

8 files changed

Lines changed: 595 additions & 7 deletions

File tree

‎src/openjd/model/_format_strings/_expression.py‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,24 @@ def evaluate(
8686

8787
raise ExpressionError(f"Nonvalid result type: {result} of type {type(result)}")
8888

89-
def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = None) -> Any:
89+
@property
90+
def is_expr(self) -> bool:
91+
"""Whether this expression uses the EXPR language (Expression Language),
92+
rather than the base 2023-09 value-reference grammar."""
93+
return getattr(self._expresion_tree, "allows_nonscalar", False)
94+
95+
def evaluate_value(
96+
self,
97+
*,
98+
symtab: SymbolTable,
99+
path_format: Optional[Any] = None,
100+
target_type: Optional[Any] = None,
101+
) -> Any:
90102
"""Evaluate the expression, preferring the engine's typed value form.
91103
104+
``target_type`` (an ``openjd.expr.ExprType``), when given, applies to
105+
EXPR-backed expressions: the result is coerced toward it.
106+
92107
For EXPR-backed expressions this returns the engine ``ExprValue``,
93108
which keeps the result's EXPR type (path/int/float/list/...) and its
94109
string-rendering fidelity when re-seeded into a symbol table — the
@@ -103,6 +118,10 @@ def evaluate_value(self, *, symtab: SymbolTable, path_format: Optional[Any] = No
103118
if evaluate_value is None:
104119
return self.evaluate(symtab=symtab, path_format=path_format)
105120
try:
121+
if target_type is not None:
122+
return evaluate_value(
123+
symtab=symtab, path_format=path_format, target_type=target_type
124+
)
106125
return evaluate_value(symtab=symtab, path_format=path_format)
107126
except ValueError as exc:
108127
raise ExpressionError(f"Expression failed validation: {str(exc)}")

‎src/openjd/model/_format_strings/_format_string.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22

3+
import functools
34
from dataclasses import dataclass
45
from typing import Any, Optional, Union
56

@@ -10,6 +11,16 @@
1011
from .._types import ModelParsingContextInterface, SpecificationRevision
1112

1213

14+
@functools.lru_cache(maxsize=None)
15+
def _expr_type(type_string: str) -> Any:
16+
"""The ``openjd.expr.ExprType`` for ``type_string``. Imported here, on
17+
first use, because importing openjd.model must not load the native
18+
extension; only an EXPR template reaches the callers."""
19+
from openjd.expr import ExprType
20+
21+
return ExprType(type_string)
22+
23+
1324
class _ReconstructionContext(ModelParsingContextInterface):
1425
"""Minimal parsing context used when a FormatString is reconstructed by
1526
pickle or the copy module (see ``FormatString.__getnewargs_ex__``).
@@ -224,6 +235,55 @@ def resolve_value(self, *, symtab: SymbolTable, path_format: Optional[object] =
224235
)
225236
return self.resolve(symtab=symtab, path_format=path_format)
226237

238+
def resolve_list_item(
239+
self, *, symtab: SymbolTable, target_type: str, path_format: Optional[object] = None
240+
) -> list[Any]:
241+
"""Resolve this format string as a list item (Expression Language §1.3.2).
242+
243+
When the format string is exactly one EXPR ``{{ ... }}`` expression,
244+
with no text around it, the expression is evaluated with
245+
``target_type`` (an EXPR type string of the form ``T? | list[T]``,
246+
such as ``"int | list[int] | nulltype"``): a ``null`` result contributes no values, a list
247+
contributes one value per member, and any other result contributes
248+
itself. The values are the engine's ``ExprValue``s. A result that does
249+
not coerce to the target raises.
250+
251+
Any other format string (text around its expressions, several
252+
expressions, or a template without EXPR, whose expressions are bare
253+
value references that cannot produce ``null`` or a list) contributes
254+
its resolved string, as :meth:`resolve` returns it.
255+
256+
Raises:
257+
FormatStringError: if the expression cannot be evaluated or its
258+
result does not coerce to ``target_type``.
259+
"""
260+
expressions = self.expressions
261+
if len(expressions) == 1:
262+
info = expressions[0]
263+
expression = info.expression
264+
whole_field = info.start_pos == 0 and info.end_pos == len(self.original_value)
265+
if whole_field and expression is not None and getattr(expression, "is_expr", False):
266+
try:
267+
value = expression.evaluate_value(
268+
symtab=symtab,
269+
path_format=path_format,
270+
target_type=_expr_type(target_type),
271+
)
272+
except ExpressionError as exc:
273+
raise FormatStringError(
274+
string=self.original_value,
275+
start=info.start_pos,
276+
end=info.end_pos,
277+
expr=expression.expr,
278+
details=str(exc),
279+
)
280+
if getattr(value, "is_null", False):
281+
return []
282+
if str(value.type).startswith("list["):
283+
return list(value)
284+
return [value]
285+
return [self.resolve(symtab=symtab, path_format=path_format)]
286+
227287
def _preprocess(
228288
self, *, context: ModelParsingContextInterface
229289
) -> list[Union[str, ExpressionInfo]]:

‎src/openjd/model/_format_strings/_nodes.py‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -220,12 +220,17 @@ def _check_comprehension_shadowing(self, symbols: set) -> None:
220220
"a variable that already exists at this location."
221221
)
222222

223-
def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any:
223+
def _evaluate_raw(
224+
self, *, symtab: SymbolTable, path_format: Any = None, target_type: Any = None
225+
) -> Any:
224226
"""Evaluate the expression and return the engine's ``ExprValue``.
225227
226228
Shared by :meth:`evaluate` (which unwraps to the native Python value)
227229
and :meth:`evaluate_to_str` (which uses the engine's spec-defined string
228230
coercion). Re-raises engine errors as the model's ``ExpressionError``.
231+
``target_type`` (an ``openjd.expr.ExprType``), when given, is the
232+
result's target type (Expression Language §1.2.3): the result is
233+
coerced toward it, and evaluation fails when it cannot be.
229234
"""
230235
from ._expr_support import (
231236
map_eval_error,
@@ -242,16 +247,20 @@ def _evaluate_raw(self, *, symtab: SymbolTable, path_format: Any = None) -> Any:
242247
)
243248
profile = profile_for_symtab(symtab)
244249
try:
245-
return self._parsed.evaluate(values=values, profile=profile, path_format=path_format)
250+
return self._parsed.evaluate(
251+
values=values, profile=profile, path_format=path_format, target_type=target_type
252+
)
246253
except Exception as exc: # noqa: BLE001 - boundary; re-raise as model error
247254
raise map_eval_error(exc)
248255

249256
def evaluate(self, *, symtab: SymbolTable, path_format: Any = None) -> Any:
250257
return self._evaluate_raw(symtab=symtab, path_format=path_format).item()
251258

252-
def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any:
259+
def evaluate_value(
260+
self, *, symtab: SymbolTable, path_format: Any = None, target_type: Any = None
261+
) -> Any:
253262
"""Evaluate and return the engine's ``ExprValue`` (not unwrapped to a
254-
native Python value).
263+
native Python value), coerced toward ``target_type`` when given.
255264
256265
Callers that re-seed the result into a symbol table (e.g. the session
257266
runtime's ``let`` bindings, RFC 0007) should use this form: the typed
@@ -261,7 +270,7 @@ def evaluate_value(self, *, symtab: SymbolTable, path_format: Any = None) -> Any
261270
are preserved), matching the Rust runtime's natively typed symbol
262271
table.
263272
"""
264-
return self._evaluate_raw(symtab=symtab, path_format=path_format)
273+
return self._evaluate_raw(symtab=symtab, path_format=path_format, target_type=target_type)
265274

266275
def evaluate_to_str(self, *, symtab: SymbolTable, path_format: Any = None) -> str:
267276
# ``str(ExprValue)`` applies the engine's RFC 0005 format-string

‎src/openjd/model/_internal/_create_job.py‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
from .._symbol_table import SymbolTable
1111
from .._format_strings import FormatString
12-
from .._types import ModelParsingContextInterface, OpenJDModel
12+
from .._types import ListItemSpec, ModelParsingContextInterface, OpenJDModel
1313

1414
__all__ = ("instantiate_model", "resolve_whole_field_typed_list")
1515

@@ -262,6 +262,15 @@ def _instantiate_model( # noqa: C901
262262
instantiated = coerce(model, field_name, raw_typed)
263263
else:
264264
instantiated = unwrap_typed_resolution(raw_typed)
265+
elif (
266+
isinstance(field_value, list)
267+
and field_name in model._job_creation_metadata.list_item_fields
268+
):
269+
instantiated = _instantiate_list_items(
270+
field_value,
271+
symtab,
272+
model._job_creation_metadata.list_item_fields[field_name],
273+
)
265274
elif isinstance(field_value, list):
266275
if field_name in model._job_creation_metadata.reshape_field_to_dict:
267276
key_field = model._job_creation_metadata.reshape_field_to_dict[field_name]
@@ -366,6 +375,45 @@ def _instantiate_list_field_as_list( # noqa: C901
366375
return result
367376

368377

378+
def _instantiate_list_items(
379+
value: list[Any],
380+
symtab: SymbolTable,
381+
spec: ListItemSpec,
382+
) -> list[Any]:
383+
"""Instantiate a list whose format string elements are list items
384+
(Expression Language §1.3.2; see ``JobCreationMetadata.list_item_fields``).
385+
386+
Each format string element contributes the values
387+
:meth:`FormatString.resolve_list_item` resolves it to, and every value,
388+
and every element that is not a format string, goes through
389+
``spec.convert``. Errors are reported at the element's index in the list
390+
as written.
391+
392+
Raises:
393+
ValidationError: if an element fails to resolve or convert.
394+
ValueError: if a list that was not empty resolves to no elements.
395+
"""
396+
errors: list[InitErrorDetails] = []
397+
result: list[Any] = []
398+
for idx, item in enumerate(value):
399+
with capture_validation_errors(output_errors=errors, loc=(idx,), input=value):
400+
if isinstance(item, FormatString):
401+
resolved = item.resolve_list_item(symtab=symtab, target_type=spec.target_type)
402+
else:
403+
resolved = [item]
404+
result.extend(spec.convert(v) for v in resolved)
405+
406+
if errors:
407+
raise ValidationError.from_exception_data(
408+
title="_instantiate_list_items", line_errors=errors
409+
)
410+
# The non-empty rule is checked on the list as written, at decode; dropped
411+
# elements can still leave it empty.
412+
if value and not result:
413+
raise ValueError("has no elements after resolution.")
414+
return result
415+
416+
369417
def _instantiate_list_field_as_dict( # noqa: C901
370418
value: list[Any],
371419
symtab: SymbolTable,

‎src/openjd/model/_types.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,24 @@ class JobCreateAsMetadata:
258258
callable: Optional[Callable[["OpenJDModel", dict], Type["OpenJDModel"]]] = field(default=None)
259259

260260

261+
@dataclass(frozen=True, eq=False, **dataclass_kwargs)
262+
class ListItemSpec:
263+
"""How the elements of a list field resolve as list items (see
264+
``JobCreationMetadata.list_item_fields``).
265+
266+
Attributes:
267+
target_type: The EXPR type string of a whole-field EXPR element's
268+
target, of the form ``T? | list[T]``, such as
269+
``"int | list[int] | nulltype"``.
270+
convert: Turns one resolved value, an ``openjd.expr.ExprValue`` from a
271+
typed element or the ``str`` of any other element, into the
272+
instantiated element. Raises ``ValueError`` to reject it.
273+
"""
274+
275+
target_type: str
276+
convert: Callable[[Any], Any]
277+
278+
261279
@dataclass(frozen=True, eq=False, **dataclass_kwargs)
262280
class JobCreationMetadata:
263281
"""Metadata for creating a job out of a job template.
@@ -302,6 +320,19 @@ class JobCreationMetadata:
302320
native Python list unchanged.
303321
"""
304322

323+
list_item_fields: dict[str, "ListItemSpec"] = field(default_factory=dict)
324+
"""List fields (a subset of ``resolve_fields``) whose format string
325+
elements are list items (Expression Language §1.3.2), keyed by field name.
326+
Each element resolves through :meth:`FormatString.resolve_list_item` with
327+
the spec's target type, so with EXPR a whole-field ``null`` drops the
328+
element and a list flattens into one element per member. The spec's
329+
``convert`` turns each resolved value into the instantiated element, and
330+
may raise ``ValueError`` to reject it; the error is reported at the
331+
element's index in the list as written. A list that was not empty as
332+
written but resolves to no elements is rejected. Used by task-parameter
333+
``range`` lists.
334+
"""
335+
305336
create_as: Optional[JobCreateAsMetadata] = field(default=None)
306337
"""The class to create the model as during instantiation. Options:
307338
1. None -- create as the model class itself.

0 commit comments

Comments
 (0)