Skip to content

Commit 812b351

Browse files
author
nayan3107
committed
feat(mlops/workflow): Align ForEachStep with service
The service still resolves the ForEach aggregate members (Status, InputItemCount and the step counts) as downstream references, and accepts a literal IterableItems list of up to 100 scalars or property references. Expose both and add ForEach to the step-type guard test. --- X-AI-Prompt: Fix the ForEachStep gaps against the Tioga backend X-AI-Tool: kiro-cli
1 parent 872d45f commit 812b351

3 files changed

Lines changed: 136 additions & 29 deletions

File tree

‎sagemaker-mlops/src/sagemaker/mlops/workflow/for_each_step.py‎

Lines changed: 58 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
MAX_CONCURRENCY_MIN = 1
2727
MAX_CONCURRENCY_MAX = 50
28+
MAX_LITERAL_ITERABLE_ITEMS = 100
2829

2930
_ITERATION_OUTPUT_NAMESPACE = "IterationOutput"
3031
_CURRENT_ITEM_PATH = "ForEach.CurrentItem"
@@ -229,12 +230,29 @@ class ForEachStep(Step):
229230
- ``properties.Selected.<member>`` — output of the selector-picked
230231
iteration (no body step name qualifier: the selector's ``field`` already
231232
identifies which body step's output was compared)
233+
- ``properties.Status`` (String: Succeeded | Failed | Stopped)
234+
- ``properties.InputItemCount`` (Integer: number of items iterated) and
235+
``properties.TotalStepCount``, ``properties.SucceededStepCount``,
236+
``properties.FailedStepCount``, ``properties.StoppedStepCount``
237+
(Integers: child step counts across all iterations, so up to
238+
``InputItemCount`` times the number of body steps)
232239
"""
233240

241+
_AGGREGATE_PROPERTY_MEMBERS = (
242+
"Status",
243+
"InputItemCount",
244+
"TotalStepCount",
245+
"SucceededStepCount",
246+
"FailedStepCount",
247+
"StoppedStepCount",
248+
)
249+
234250
def __init__(
235251
self,
236252
name: str,
237-
iterable_items: PipelineVariable = None,
253+
iterable_items: Union[
254+
PipelineVariable, List[Union[PipelineVariable, str, int, float, bool]]
255+
] = None,
238256
max_concurrency: int = None,
239257
for_each_body: List[Step] = None,
240258
selector: Optional[ForEachSelector] = None,
@@ -246,15 +264,18 @@ def __init__(
246264
247265
Args:
248266
name (str): The name of the `ForEachStep`.
249-
iterable_items (PipelineVariable): REQUIRED. The items to iterate
250-
over. A property reference (e.g. a `ParameterString` or a step
251-
output) whose value must resolve to a JSON list string at
252-
runtime, such as ``'["item-a", "item-b"]'``. The service
253-
currently only accepts the property-reference form; literal
254-
values (including literal lists) are rejected ("a literal value
255-
cannot resolve to a list"). Items are commonly JSON-object
256-
strings whose fields are addressed via ``CurrentItem().<field>``
257-
inside the body step.
267+
iterable_items (Union[PipelineVariable, List]): REQUIRED. The items
268+
to iterate over, in one of two forms:
269+
270+
- A property reference (e.g. a `ParameterString` or a step
271+
output) whose value resolves to a JSON list string at
272+
runtime, such as ``'["item-a", "item-b"]'``.
273+
- A non-empty list of at most 100 elements, each a property
274+
reference or a literal scalar (str, int, float, bool). Each
275+
element is one iteration item.
276+
277+
Items are commonly JSON-object strings whose fields are
278+
addressed via ``CurrentItem().<field>`` inside the body step.
258279
max_concurrency (int): REQUIRED. The maximum number of child steps
259280
that run concurrently. Range: [1, 50].
260281
for_each_body (List[Step]): REQUIRED. The step(s) to run once per
@@ -279,12 +300,16 @@ def __init__(
279300
self._validate_for_each_body(name, for_each_body)
280301
self._validate_selector(name, selector, for_each_body)
281302

282-
self.iterable_items = iterable_items
303+
self.iterable_items = (
304+
list(iterable_items) if isinstance(iterable_items, list) else iterable_items
305+
)
283306
self.max_concurrency = max_concurrency
284307
self.for_each_body = list(for_each_body)
285308
self.selector = selector
286309

287310
root_prop = Properties(step_name=name)
311+
for member in ForEachStep._AGGREGATE_PROPERTY_MEMBERS:
312+
root_prop.__dict__[member] = Properties(step_name=name, path=member, step=self)
288313
root_prop.__dict__["IterationOutputs"] = _IterationOutputsAccessor(self)
289314
root_prop.__dict__["Selected"] = _ForEachPropertyReference(f"Steps.{name}.Selected", [self])
290315
self._properties = root_prop
@@ -296,13 +321,29 @@ def _validate_iterable_items(name: str, iterable_items: Any):
296321
raise ValueError(
297322
f"ForEach step '{name}': iterable_items is required. Provide a "
298323
f"property reference that resolves to a JSON list string at "
299-
f"runtime."
324+
f"runtime, or a list of items."
300325
)
326+
if isinstance(iterable_items, list):
327+
if not 1 <= len(iterable_items) <= MAX_LITERAL_ITERABLE_ITEMS:
328+
raise ValueError(
329+
f"ForEach step '{name}': an iterable_items list must have between 1 "
330+
f"and {MAX_LITERAL_ITERABLE_ITEMS} elements. Got: {len(iterable_items)}."
331+
)
332+
for element in iterable_items:
333+
if element is None or not isinstance(
334+
element, (PipelineVariable, str, int, float, bool)
335+
):
336+
raise ValueError(
337+
f"ForEach step '{name}': iterable_items list elements must be "
338+
f"property references or literal scalars (str, int, float, bool). "
339+
f"Got: {type(element)}."
340+
)
341+
return
301342
if not isinstance(iterable_items, PipelineVariable):
302343
raise ValueError(
303344
f"ForEach step '{name}': iterable_items must be a property "
304-
f"reference that resolves to a JSON list string at runtime; "
305-
f"a literal value cannot resolve to a list. "
345+
f"reference that resolves to a JSON list string at runtime, or a "
346+
f"list of items; a single literal value cannot resolve to a list. "
306347
f"Got: {type(iterable_items)}."
307348
)
308349

@@ -398,6 +439,8 @@ def step_only_arguments(self) -> RequestType:
398439
def properties(self):
399440
"""A Properties object exposing the ForEach step's results.
400441
401-
Members: ``IterationOutputs[i].<bodyStepName>`` and ``Selected``.
442+
Members: ``Status``, ``InputItemCount``, ``TotalStepCount``,
443+
``SucceededStepCount``, ``FailedStepCount``, ``StoppedStepCount``,
444+
``IterationOutputs[i].<bodyStepName>`` and ``Selected``.
402445
"""
403446
return self._properties

‎sagemaker-mlops/tests/unit/workflow/test_for_each_step.py‎

Lines changed: 77 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -121,9 +121,7 @@ def test_current_item_drilled_expr():
121121
def test_current_item_bracket_indexing():
122122
assert CurrentItem()["score"].expr == {"Get": "ForEach.CurrentItem['score']"}
123123
assert CurrentItem().items[0].expr == {"Get": "ForEach.CurrentItem.items[0]"}
124-
assert CurrentItem()["nested"]["key"].expr == {
125-
"Get": "ForEach.CurrentItem['nested']['key']"
126-
}
124+
assert CurrentItem()["nested"]["key"].expr == {"Get": "ForEach.CurrentItem['nested']['key']"}
127125

128126

129127
def test_current_item_has_no_referenced_steps():
@@ -163,12 +161,23 @@ def test_for_each_step_to_request_reference_items(item_list_parameter):
163161
assert args["ForEachBody"][0]["Type"] == "Lambda"
164162

165163

166-
def test_for_each_step_literal_array_items_rejected():
167-
# the service currently only accepts the property-reference form
168-
literal_item = '{"name": "lit-c", "score": 5}'
164+
def test_for_each_step_literal_list_items_accepted():
169165
ref_item = ParameterString(name="ItemOne")
170-
with pytest.raises(ValueError, match="literal value cannot resolve to a list"):
171-
make_for_each_step(iterable_items=[ref_item, literal_item])
166+
items = [ref_item, '{"name": "lit-c", "score": 5}', 42, 1.5, True]
167+
step = make_for_each_step(iterable_items=items)
168+
assert step.to_request()["Arguments"]["IterableItems"] == items
169+
170+
171+
@pytest.mark.parametrize("items", [[], ["x"] * 101])
172+
def test_for_each_step_literal_list_size_out_of_range(items):
173+
with pytest.raises(ValueError, match="between 1 and 100 elements"):
174+
make_for_each_step(iterable_items=items)
175+
176+
177+
@pytest.mark.parametrize("element", [None, {"a": 1}, ["nested"]])
178+
def test_for_each_step_literal_list_invalid_element(element):
179+
with pytest.raises(ValueError, match="literal scalars"):
180+
make_for_each_step(iterable_items=["ok", element])
172181

173182

174183
def test_for_each_step_to_request_with_selector():
@@ -219,6 +228,24 @@ def test_for_each_step_iteration_outputs_property():
219228
assert ref._referenced_steps == [step]
220229

221230

231+
@pytest.mark.parametrize(
232+
"member",
233+
[
234+
"Status",
235+
"InputItemCount",
236+
"TotalStepCount",
237+
"SucceededStepCount",
238+
"FailedStepCount",
239+
"StoppedStepCount",
240+
],
241+
)
242+
def test_for_each_step_aggregate_properties(member):
243+
step = make_for_each_step()
244+
ref = getattr(step.properties, member)
245+
assert ref.expr == {"Get": f"Steps.ForEachItems.{member}"}
246+
assert ref._referenced_steps == [step]
247+
248+
222249
def test_for_each_step_iteration_outputs_must_be_indexed():
223250
step = make_for_each_step()
224251
with pytest.raises(TypeError, match="must be indexed"):
@@ -228,9 +255,7 @@ def test_for_each_step_iteration_outputs_must_be_indexed():
228255
def test_for_each_step_selected_property():
229256
step = make_for_each_step()
230257
ref = step.properties.Selected.OutputParameters["ProcessedScore"]
231-
assert ref.expr == {
232-
"Get": "Steps.ForEachItems.Selected.OutputParameters['ProcessedScore']"
233-
}
258+
assert ref.expr == {"Get": "Steps.ForEachItems.Selected.OutputParameters['ProcessedScore']"}
234259
assert ref._referenced_steps == [step]
235260

236261

@@ -451,9 +476,7 @@ def test_pipeline_definition_downstream_references(mock_session):
451476
"Get": "Steps.ForEachItems.IterationOutputs[0].ProcessItem"
452477
".OutputParameters['ResultPath']"
453478
},
454-
"selected": {
455-
"Get": "Steps.ForEachItems.Selected.OutputParameters['ResultPath']"
456-
},
479+
"selected": {"Get": "Steps.ForEachItems.Selected.OutputParameters['ResultPath']"},
457480
}
458481
# dependency is inferred by the service from the Get expressions;
459482
# no explicit DependsOn is added for property references
@@ -539,3 +562,43 @@ def test_pipeline_graph_includes_body_step_edges(mock_session):
539562
graph = PipelineGraph([for_each])
540563
assert "ProcessItem" in graph.step_map
541564
assert "ProcessItem" in graph.adjacency_list["ForEachItems"]
565+
566+
567+
def test_pipeline_definition_literal_list_items_and_aggregate_reference(mock_session):
568+
body = CustomStep(name="ProcessItem", input_data=CurrentItem())
569+
producer = CustomStep(name="ProduceItem")
570+
producer_output = Properties(
571+
step_name="ProduceItem", path="OutputParameters['ItemA']", step=producer
572+
)
573+
for_each = ForEachStep(
574+
name="ForEachItems",
575+
iterable_items=[producer_output, "s3://my-bucket/item-b", 42, True],
576+
max_concurrency=2,
577+
for_each_body=[body],
578+
)
579+
consumer = CustomStep(
580+
name="Consumer",
581+
input_data={
582+
"status": for_each.properties.Status,
583+
"total": for_each.properties.TotalStepCount,
584+
},
585+
)
586+
pipeline = Pipeline(
587+
name="test-pipeline",
588+
steps=[producer, for_each, consumer],
589+
sagemaker_session=mock_session,
590+
)
591+
steps = _definition_steps(pipeline)
592+
for_each_request = next(s for s in steps if s["Name"] == "ForEachItems")
593+
# same shape as the service's iterable_literal_list.json fixture
594+
assert for_each_request["Arguments"]["IterableItems"] == [
595+
{"Get": "Steps.ProduceItem.OutputParameters['ItemA']"},
596+
"s3://my-bucket/item-b",
597+
42,
598+
True,
599+
]
600+
consumer_request = next(s for s in steps if s["Name"] == "Consumer")
601+
assert consumer_request["Arguments"]["input_data"] == {
602+
"status": {"Get": "Steps.ForEachItems.Status"},
603+
"total": {"Get": "Steps.ForEachItems.TotalStepCount"},
604+
}

‎sagemaker-mlops/tests/unit/workflow/test_job_step.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
"INFERENCE_COMPONENT": "InferenceComponent",
5151
"LINEAGE": "Lineage",
5252
"JOB": "Job",
53+
"FOR_EACH": "ForEach",
5354
}
5455

5556

0 commit comments

Comments
 (0)