Skip to content

Commit f6bf876

Browse files
authored
fix: Allow underscores in a combination expression (#367)
* fix: Allow '_' in a combination expression Template Schemas §3.4.3 constraint 1 gives a <CombinationExpr> the characters allowed in an <Identifier> plus the space and the operators, and §7.1 puts the underscore among an <Identifier>'s characters. The character class here omitted it, so a task parameter named Frame_Range was rejected by the pattern before the expression parser saw it: steps[0] -> parameterSpace -> combination: String should match pattern '(?-m:^[A-Za-z0-9\*\(\), ]+\z)' The name itself was legal, so such a parameter could be declared and never referenced. Its own comment already claimed the intended relationship to <Identifier>, so bind both classes to one constant rather than restating it: the two cannot drift again. The rendered _identifier_regex is unchanged. The Rust path accepted these all along, so the two lanes disagreed. Add a control on the v1 side pinning its half of the parity. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> * fix: Address review on the combination expression class Two findings from the automated review. Scope the comment's claim. It said the shared constant meant Identifier and CombinationExpr "cannot drift", which is stronger than the code provides: only the character class is shared, not the leading-character rule. Say what is actually shared, and record why the space is U+0020 alone. Whitespace stays narrow, deliberately. The shared TokenStream folds every whitespace run to a space before lexing, so CombinationExpressionParser on its own accepts a newline that the pattern refuses. §3.4.3 allows "the space", and openjd-rs refuses a newline too, so the pattern is the conformant side and widening it to \s would create a divergence rather than close one. Pin it with a newline case instead of leaving the decision implicit. Assert the shared constant, not the rendered pattern. Hardcoding the expanded regex meant a deliberate widening of _identifier_chars would break a test about hyphens, which is the opposite of what factoring the constant out was for. Verified: widening the constant now leaves the negative controls green, while reverting it, widening to any character, and widening the space to \s are each caught. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --------- Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com>
1 parent 9a56a9b commit f6bf876

3 files changed

Lines changed: 167 additions & 5 deletions

File tree

‎src/openjd/model/v2023_09/_model.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,9 @@ class ValueReferenceConstants(Enum):
193193
_standard_string_regex = rf"(?-m:^[^{_Cc_characters}]+\z)"
194194

195195
# Latin alphanumeric, starting with a letter
196-
_identifier_regex = r"(?-m:^[A-Za-z_][A-Za-z0-9_]*\z)"
196+
# Shared with CombinationExpr, which §3.4.3 defines in terms of these characters.
197+
_identifier_chars = r"A-Za-z0-9_"
198+
_identifier_regex = rf"(?-m:^[A-Za-z_][{_identifier_chars}]*\z)"
197199

198200
# Regex for defining file filter patterns allowed for use in file dialogs.
199201
# 1. Allowable values: "*", "*.*", and "*.[:file-extension-chars:]+".
@@ -1777,12 +1779,17 @@ def _validate_range_elements(cls, value: Any) -> Any:
17771779
max_length=16,
17781780
),
17791781
]
1780-
# Limit the CombinationExpr to characters allowed in an Identifier plus whitespace
1781-
# and the operator characters.
1782+
# §3.4.3: an Identifier's characters plus the space and the operators. Only the
1783+
# character class is shared with Identifier, not its leading-character rule.
1784+
# The space is deliberately just U+0020. The shared TokenStream folds all
1785+
# whitespace, which is wider than §3.4.3 allows, so a newline is refused here.
17821786
CombinationExpr = Annotated[
17831787
str,
17841788
StringConstraints(
1785-
min_length=1, max_length=1280, strict=True, pattern=r"(?-m:^[A-Za-z0-9\*\(\), ]+\z)"
1789+
min_length=1,
1790+
max_length=1280,
1791+
strict=True,
1792+
pattern=rf"(?-m:^[{_identifier_chars}\*\(\), ]+\z)",
17861793
),
17871794
]
17881795

‎test/openjd/model_v0/v2023_09/test_parameter_space.py‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,13 @@
66
import pytest
77
from pydantic import ValidationError
88

9-
from openjd.model import DecodeValidationError, decode_job_template, parse_model
9+
from openjd.model import (
10+
DecodeValidationError,
11+
StepParameterSpaceIterator,
12+
create_job,
13+
decode_job_template,
14+
parse_model,
15+
)
1016
from openjd.model._parse import _parse_model
1117
from openjd.model.v2023_09 import (
1218
FloatTaskParameterDefinition,
@@ -17,6 +23,7 @@
1723
StepParameterSpaceDefinition,
1824
StringTaskParameterDefinition,
1925
)
26+
from openjd.model.v2023_09._model import _identifier_chars
2027

2128

2229
class TestIntTaskParameterDefinition:
@@ -1071,3 +1078,132 @@ def test_two_miscased_chunk_int_report_the_one_chunk_rule(self) -> None:
10711078
],
10721079
}
10731080
)
1081+
1082+
1083+
class TestCombinationExprCharacterClass:
1084+
"""Template Schemas §3.4.3 constraint 1 gives a combination expression the
1085+
characters of an ``<Identifier>`` plus the space and the operators. §7.1
1086+
puts ``_`` in an ``<Identifier>``, so a parameter named ``Frame_Range`` must
1087+
be referenceable. The character class omitted ``_``, which made every such
1088+
name unusable: the name itself parsed, and the reference to it did not.
1089+
1090+
The rejection landed on the pattern, before the expression parser ran, so
1091+
these go through the model rather than
1092+
``openjd.model._internal._combination_expr.Parser`` (which accepted ``_``
1093+
all along).
1094+
1095+
Whitespace is the other place the pattern and that parser disagree, and it is
1096+
left disagreeing on purpose: §3.4.3 allows "the space", so U+0020 only, while
1097+
the shared ``TokenStream`` folds every whitespace run to a space before
1098+
lexing. Widening the class to ``\\s`` would accept a multi-line expression
1099+
that openjd-rs rejects, so the pattern stays the narrower, conformant side.
1100+
``test_disallowed_characters_still_rejected`` pins that.
1101+
"""
1102+
1103+
@staticmethod
1104+
def _space(names: list[str], combination: str) -> dict[str, Any]:
1105+
return {
1106+
"taskParameterDefinitions": [
1107+
{"name": name, "type": "INT", "range": [1, 2]} for name in names
1108+
],
1109+
"combination": combination,
1110+
}
1111+
1112+
@pytest.mark.parametrize(
1113+
"names,combination",
1114+
(
1115+
pytest.param(
1116+
["Frame_Range", "Quality"], "Frame_Range * Quality", id="interior underscore"
1117+
),
1118+
pytest.param(["_Frame", "_Quality"], "(_Frame, _Quality)", id="leading underscore"),
1119+
pytest.param(["A_", "B_"], "A_ * B_", id="trailing underscore"),
1120+
pytest.param(["_", "A"], "_ * A", id="name is a bare underscore"),
1121+
pytest.param(
1122+
["A_1", "B_2", "C_3"], "A_1 * ( B_2, C_3 )", id="underscore inside an association"
1123+
),
1124+
),
1125+
)
1126+
def test_underscore_names_accepted(self, names: list[str], combination: str) -> None:
1127+
# WHEN
1128+
model = _parse_model(
1129+
model=StepParameterSpaceDefinition, obj=self._space(names, combination)
1130+
)
1131+
1132+
# THEN the expression is carried through verbatim
1133+
assert model.combination == combination
1134+
1135+
@pytest.mark.parametrize(
1136+
"combination",
1137+
(
1138+
pytest.param("Frame-Range * Quality", id="hyphen"),
1139+
pytest.param("Frame.Range * Quality", id="dot"),
1140+
pytest.param("Frame+Range * Quality", id="plus"),
1141+
pytest.param("Frame\tRange * Quality", id="tab"),
1142+
pytest.param("Frame *\nRange * Quality", id="newline"),
1143+
),
1144+
)
1145+
def test_disallowed_characters_still_rejected(self, combination: str) -> None:
1146+
# Negative control. Widening the class to admit '_' must not admit
1147+
# anything else, and the rejection must come from the pattern rather than
1148+
# from the expression parser downstream of it.
1149+
#
1150+
# The tab and newline cases are deliberate, not incidental: §3.4.3 allows
1151+
# "the space", and the shared TokenStream folds all whitespace to U+0020
1152+
# before lexing, so ``CombinationExpressionParser`` on its own accepts
1153+
# both. This is the narrower, conformant side of that disagreement, and it
1154+
# is what openjd-rs does too.
1155+
# WHEN
1156+
with pytest.raises(ValidationError) as excinfo:
1157+
_parse_model(
1158+
model=StepParameterSpaceDefinition,
1159+
obj=self._space(["Frame", "Range", "Quality"], combination),
1160+
)
1161+
1162+
# THEN it failed on the character class, and that class is the shared
1163+
# constant. Asserting the constant rather than the rendered pattern keeps
1164+
# a deliberate widening of it from breaking a test about hyphens.
1165+
message = str(excinfo.value)
1166+
assert "combination" in message, message
1167+
assert "String should match pattern" in message, message
1168+
assert _identifier_chars in message, message
1169+
1170+
def test_underscore_name_iterates_the_full_parameter_space(self) -> None:
1171+
# The character class was the only gate, so a template that clears it must
1172+
# produce the same space as one with underscore-free names: 3 x 2 = 6 tasks
1173+
# keyed by the underscore-bearing name.
1174+
# GIVEN
1175+
template = decode_job_template(
1176+
template={
1177+
"specificationVersion": "jobtemplate-2023-09",
1178+
"name": "T",
1179+
"steps": [
1180+
{
1181+
"name": "S",
1182+
"parameterSpace": {
1183+
"taskParameterDefinitions": [
1184+
{"name": "Frame_Range", "type": "INT", "range": "1-3"},
1185+
{"name": "Quality", "type": "STRING", "range": ["low", "high"]},
1186+
],
1187+
"combination": "Frame_Range * Quality",
1188+
},
1189+
"script": {"actions": {"onRun": {"command": "echo", "args": ["hi"]}}},
1190+
}
1191+
],
1192+
}
1193+
)
1194+
job = create_job(job_template=template, job_parameter_values={})
1195+
1196+
# WHEN
1197+
space = StepParameterSpaceIterator(space=job.steps[0].parameterSpace)
1198+
1199+
# THEN
1200+
assert space.names == {"Frame_Range", "Quality"}
1201+
tasks = [(params["Frame_Range"].value, params["Quality"].value) for params in space]
1202+
assert tasks == [
1203+
("1", "low"),
1204+
("1", "high"),
1205+
("2", "low"),
1206+
("2", "high"),
1207+
("3", "low"),
1208+
("3", "high"),
1209+
]

‎test/openjd/model_v1/test_step_param_space_def.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,25 @@ def test_combination_string(self) -> None:
112112
)
113113
assert step.parameter_space.combination == "A * B"
114114

115+
def test_combination_accepts_underscore_names(self) -> None:
116+
"""Template Schemas §3.4.3 constraint 1 gives a combination expression
117+
the characters of an ``<Identifier>``, and §7.1 puts ``_`` among them.
118+
119+
Control for the v0 side, where the character class omitted ``_`` and made
120+
such names unreferenceable. This path already accepted them, so the two
121+
lanes disagreed; this pins the v1 half of the parity.
122+
"""
123+
step = _decode_step(
124+
{
125+
"taskParameterDefinitions": [
126+
{"name": "Frame_Range", "type": "INT", "range": [1, 2]},
127+
{"name": "_Quality", "type": "STRING", "range": ["x", "y"]},
128+
],
129+
"combination": "Frame_Range * _Quality",
130+
}
131+
)
132+
assert step.parameter_space.combination == "Frame_Range * _Quality"
133+
115134
def test_camelcase_alias(self) -> None:
116135
"""``taskParameterDefinitions`` is a camelCase alias for
117136
``task_parameter_definitions``."""

0 commit comments

Comments
 (0)