Skip to content

Commit 953e66f

Browse files
author
Alex Wang
committed
fix(insight): validate S3 partitioning
Address the S3 partition-validation review comment on PR #632. Add a public S3Partitioning(StrEnum) (DATE=date, FUNCTION_NAME=function-name, NONE=none) in the s3_exporter module. The constructor is typed as an S3Partitioning | Literal[...] union (never bare str) and normalizes input with S3Partitioning(partitioning), so an invalid dynamic value (e.g. function_name) raises ValueError at construction instead of silently falling through to no partitioning. Key building now compares enum members. Re-export S3Partitioning from the exporters package and top-level package alongside S3Exporter. Existing API-compatible string inputs are preserved. Scheduler/flush/queueing/draining behavior is intentionally unchanged.
1 parent e268a9c commit 953e66f

4 files changed

Lines changed: 117 additions & 6 deletions

File tree

packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from aws_durable_execution_sdk_python_insight.exporters import (
88
LambdaLogExporter,
99
S3Exporter,
10+
S3Partitioning,
1011
)
1112
from aws_durable_execution_sdk_python_insight.operations_index import (
1213
build_operations_by_name,
@@ -38,6 +39,7 @@
3839
"OperationDetail",
3940
"OperationOverride",
4041
"S3Exporter",
42+
"S3Partitioning",
4143
"WorkflowInsightConfig",
4244
"WorkflowInsightPlugin",
4345
"build_operations_by_name",

packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/exporters/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@
2727
)
2828
from aws_durable_execution_sdk_python_insight.exporters.s3_exporter import (
2929
S3Exporter,
30+
S3Partitioning,
3031
)
3132

3233

3334
__all__ = [
3435
"LambdaLogExporter",
3536
"S3Exporter",
37+
"S3Partitioning",
3638
]

packages/aws-durable-execution-sdk-python-insight/src/aws_durable_execution_sdk_python_insight/exporters/s3_exporter.py

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,34 @@
55

66
from __future__ import annotations
77

8-
from typing import Any
8+
from enum import StrEnum
9+
from typing import Any, Literal
910

1011
from aws_durable_execution_sdk_python_insight.exporters._common import (
1112
compact_dumps,
1213
sanitize,
1314
)
1415

1516

17+
class S3Partitioning(StrEnum):
18+
"""S3 key partitioning scheme. The values match the JS S3 exporter's options."""
19+
20+
# year=YYYY/month=MM/day=DD/ derived from the record startTime (default)
21+
DATE = "date"
22+
# function=<functionName>/
23+
FUNCTION_NAME = "function-name"
24+
# no partition prefix
25+
NONE = "none"
26+
27+
28+
# Accepted string inputs, kept in lockstep with the enum values above. The
29+
# constructor is typed as this ``Literal`` union (never bare ``str``) so a typoed
30+
# scheme fails a static type check, while ``S3Partitioning(partitioning)`` in
31+
# ``__init__`` normalizes any accepted value to the matching enum member and
32+
# raises ``ValueError`` for an invalid dynamic string.
33+
S3PartitioningInput = Literal["date", "function-name", "none"]
34+
35+
1636
class S3Exporter:
1737
"""Writes canonical ``operations``-array records to S3.
1838
@@ -25,14 +45,18 @@ def __init__(
2545
self,
2646
bucket: str,
2747
prefix: str = "workflow-insight/",
28-
partitioning: str = "date",
48+
partitioning: S3Partitioning | S3PartitioningInput = S3Partitioning.DATE,
2949
region: str | None = None,
3050
max_record_size_bytes: int | None = None,
3151
client: Any = None,
3252
) -> None:
3353
self.bucket = bucket
3454
self.prefix = prefix
35-
self.partitioning = partitioning
55+
# ``S3Partitioning(x)`` is idempotent for members, accepts the exact
56+
# JS-style strings, and raises ``ValueError`` for an unrecognized dynamic
57+
# string so an invalid scheme fails at construction rather than silently
58+
# falling through to no partitioning.
59+
self.partitioning = S3Partitioning(partitioning)
3660
self.max_record_size_bytes = (
3761
5_000_000 if max_record_size_bytes is None else max_record_size_bytes
3862
)
@@ -70,9 +94,9 @@ def _build_key(self, record: dict[str, Any]) -> str:
7094
return f"{self.prefix}{self._partition(record)}{file_name}"
7195

7296
def _partition(self, record: dict[str, Any]) -> str:
73-
if self.partitioning == "function-name":
97+
if self.partitioning == S3Partitioning.FUNCTION_NAME:
7498
return f"function={sanitize(record.get('functionName', ''))}/"
75-
if self.partitioning == "date":
99+
if self.partitioning == S3Partitioning.DATE:
76100
start = str(record.get("startTime", ""))
77101
# YYYY-MM-DD... -> year=YYYY/month=MM/day=DD/
78102
if len(start) >= 10 and start[4] == "-" and start[7] == "-":

packages/aws-durable-execution-sdk-python-insight/tests/test_exporters.py

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,25 @@
1313
import json
1414
from typing import Any
1515

16-
from aws_durable_execution_sdk_python_insight import LambdaLogExporter, S3Exporter
16+
import pytest
17+
18+
from aws_durable_execution_sdk_python_insight import (
19+
LambdaLogExporter,
20+
S3Exporter,
21+
S3Partitioning,
22+
)
1723
from aws_durable_execution_sdk_python_insight.exporters import (
1824
LambdaLogExporter as LambdaLogExporterFromPkg,
1925
)
26+
from aws_durable_execution_sdk_python_insight.exporters import (
27+
S3Partitioning as S3PartitioningFromPkg,
28+
)
2029
from aws_durable_execution_sdk_python_insight.exporters.s3_exporter import (
2130
S3Exporter as S3ExporterFromModule,
2231
)
32+
from aws_durable_execution_sdk_python_insight.exporters.s3_exporter import (
33+
S3Partitioning as S3PartitioningFromModule,
34+
)
2335

2436

2537
def _record(**kw: Any) -> dict[str, Any]:
@@ -59,6 +71,8 @@ def put_object(self, **kwargs: Any) -> None:
5971
def test_public_import_paths_resolve_same_classes():
6072
assert LambdaLogExporter is LambdaLogExporterFromPkg
6173
assert S3Exporter is S3ExporterFromModule
74+
assert S3Partitioning is S3PartitioningFromPkg
75+
assert S3Partitioning is S3PartitioningFromModule
6276

6377

6478
# -- LambdaLogExporter --------------------------------------------------------
@@ -133,3 +147,72 @@ def test_s3_key_falls_back_to_arn_then_record():
133147
# falls back to the (sanitized) executionArn
134148
assert client.puts[0]["Key"].endswith(".json")
135149
assert "arn_aws_lambda" in client.puts[0]["Key"]
150+
151+
152+
# -- S3Partitioning -----------------------------------------------------------
153+
154+
155+
def test_s3_partitioning_enum_values():
156+
# values are the exact JS-compatible strings, and StrEnum members compare
157+
# equal to those strings
158+
assert S3Partitioning.DATE == "date"
159+
assert S3Partitioning.FUNCTION_NAME == "function-name"
160+
assert S3Partitioning.NONE == "none"
161+
assert {p.value for p in S3Partitioning} == {"date", "function-name", "none"}
162+
163+
164+
def test_s3_partitioning_accepts_enum_member_input():
165+
client = FakeS3Client()
166+
exporter = S3Exporter(
167+
bucket="b", partitioning=S3Partitioning.FUNCTION_NAME, client=client
168+
)
169+
assert exporter.partitioning is S3Partitioning.FUNCTION_NAME
170+
exporter.export(_record())
171+
assert client.puts[0]["Key"] == "workflow-insight/function=my-fn/exec-1.json"
172+
173+
174+
def test_s3_partitioning_normalizes_valid_string_inputs():
175+
# existing API-compatible string inputs still work and normalize to members
176+
for value, member in (
177+
("date", S3Partitioning.DATE),
178+
("function-name", S3Partitioning.FUNCTION_NAME),
179+
("none", S3Partitioning.NONE),
180+
):
181+
exporter = S3Exporter(bucket="b", partitioning=value, client=FakeS3Client())
182+
assert exporter.partitioning is member
183+
184+
185+
def test_s3_partitioning_default_is_date():
186+
exporter = S3Exporter(bucket="b", client=FakeS3Client())
187+
assert exporter.partitioning is S3Partitioning.DATE
188+
189+
190+
def test_s3_partitioning_date_key_layout():
191+
client = FakeS3Client()
192+
S3Exporter(bucket="b", partitioning="date", client=client).export(_record())
193+
assert (
194+
client.puts[0]["Key"]
195+
== "workflow-insight/year=2026/month=01/day=01/exec-1.json"
196+
)
197+
198+
199+
def test_s3_partitioning_function_name_key_layout():
200+
client = FakeS3Client()
201+
S3Exporter(
202+
bucket="b", partitioning="function-name", prefix="wi/", client=client
203+
).export(_record(functionName="fn/weird name", executionName="exec/1"))
204+
assert client.puts[0]["Key"] == "wi/function=fn_weird_name/exec_1.json"
205+
206+
207+
def test_s3_partitioning_none_key_layout_has_no_prefix_segment():
208+
client = FakeS3Client()
209+
S3Exporter(bucket="b", partitioning="none", client=client).export(_record())
210+
assert client.puts[0]["Key"] == "workflow-insight/exec-1.json"
211+
212+
213+
def test_s3_partitioning_invalid_string_raises_value_error():
214+
# a dynamic invalid value (e.g. underscore variant) fails at construction
215+
with pytest.raises(ValueError):
216+
S3Exporter(bucket="b", partitioning="function_name", client=FakeS3Client())
217+
with pytest.raises(ValueError):
218+
S3Exporter(bucket="b", partitioning="bogus", client=FakeS3Client())

0 commit comments

Comments
 (0)