Skip to content

Commit c0c42b5

Browse files
author
nayan3107
committed
feat(mlops): Add AthenaQueryConfig for AthenaStep
Customers should not hand-build the nested StartQueryExecution dict. AthenaQueryConfig takes named params and assembles the request, like EMRServerlessJobConfig; the service still validates it at CreatePipeline. A raw arguments dict remains as an escape hatch for fields the config does not cover. --- X-AI-Prompt: Address review asking for named params instead of a raw dict on AthenaStep X-AI-Tool: kiro-cli
1 parent 149e05e commit c0c42b5

4 files changed

Lines changed: 285 additions & 99 deletions

File tree

‎sagemaker-mlops/src/sagemaker/mlops/workflow/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444

4545
# Step implementations
4646
from sagemaker.mlops.workflow.automl_step import AutoMLStep
47-
from sagemaker.mlops.workflow.athena_step import AthenaStep
47+
from sagemaker.mlops.workflow.athena_step import AthenaQueryConfig, AthenaStep
4848
from sagemaker.mlops.workflow.callback_step import CallbackStep, CallbackOutput
4949
from sagemaker.mlops.workflow.clarify_check_step import ClarifyCheckStep
5050
from sagemaker.mlops.workflow.cleanup_step import CleanupStep, S3CleanupSource
@@ -114,6 +114,7 @@
114114
"TuningStep",
115115
# Step implementations
116116
"AutoMLStep",
117+
"AthenaQueryConfig",
117118
"AthenaStep",
118119
"CallbackStep",
119120
"CallbackOutput",

‎sagemaker-mlops/src/sagemaker/mlops/workflow/athena_step.py‎

Lines changed: 152 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -13,36 +13,35 @@
1313
"""Step definition for the Athena step type in SageMaker Pipelines.
1414
1515
``AthenaStep`` runs an Amazon Athena query as part of a SageMaker Pipelines
16-
execution. The step is a pure passthrough: ``arguments`` is the Athena
17-
``StartQueryExecution`` request expressed as a dict, and it is emitted
18-
verbatim as the step's ``Arguments`` in the pipeline definition. The
19-
SageMaker Pipelines service validates it against the Athena API shape at
20-
``CreatePipeline`` time, so the SDK adds no vocabulary of its own -- use
21-
the same PascalCase field names as the Athena API / boto3.
16+
execution. Describe the query with an ``AthenaQueryConfig``; it assembles the
17+
Athena ``StartQueryExecution`` request that becomes the step's ``Arguments``::
2218
23-
Example::
24-
25-
from sagemaker.mlops.workflow import AthenaStep, ParameterString
19+
from sagemaker.mlops.workflow import AthenaQueryConfig, AthenaStep, ParameterString
2620
2721
output_location = ParameterString(name="OutputLocation")
2822
2923
step = AthenaStep(
3024
name="RunQuery",
31-
arguments={
32-
"QueryString": "SELECT ? AS region",
33-
"QueryExecutionContext": {"Database": "default"},
34-
"ResultConfiguration": {"OutputLocation": output_location},
35-
"WorkGroup": "primary",
36-
"ExecutionParameters": ["'us-west-2'"],
37-
},
25+
query_config=AthenaQueryConfig(
26+
query_string="SELECT ? AS region",
27+
database="default",
28+
work_group="primary",
29+
output_location=output_location,
30+
encryption_option="SSE_S3",
31+
execution_parameters=["'us-west-2'"],
32+
),
3833
)
3934
40-
Notes on service-side rules (enforced at ``CreatePipeline`` / execution,
41-
not by this class):
35+
For ``StartQueryExecution`` fields that ``AthenaQueryConfig`` does not cover,
36+
pass the full request as a dict through ``arguments`` instead; it is emitted
37+
unchanged.
38+
39+
The SageMaker Pipelines service validates the request at ``CreatePipeline``
40+
time, not this module:
4241
4342
* ``QueryString`` is literal-only. Pipeline expressions such as
4443
``{"Get": "Parameters.X"}`` inside it are rejected to prevent SQL
45-
injection. Bind runtime values with ``ExecutionParameters`` (``?``
44+
injection. Bind runtime values with ``execution_parameters`` (``?``
4645
placeholders) instead; pipeline variables are allowed there and in every
4746
other field.
4847
* ``ClientRequestToken`` is generated by the service at execution time and
@@ -61,7 +60,7 @@
6160
import copy
6261
from typing import Any, Dict, List, Optional, Union
6362

64-
from sagemaker.core.helper.pipeline_variable import RequestType
63+
from sagemaker.core.helper.pipeline_variable import PipelineVariable, RequestType
6564
from sagemaker.core.workflow.properties import Properties
6665

6766
from sagemaker.mlops.workflow.retry import RetryPolicy
@@ -74,34 +73,144 @@
7473
)
7574

7675

77-
class AthenaStep(ConfigurableRetryStep):
78-
"""Runs an Amazon Athena query as part of a SageMaker Pipeline.
76+
class AthenaQueryConfig:
77+
"""Config for an Athena query run by an ``AthenaStep``.
7978
80-
``arguments`` is the Athena ``StartQueryExecution`` request as a dict and
81-
is passed through unchanged as the step's ``Arguments``. Field names,
82-
nesting, and which fields are required are all defined by the Athena API
83-
and validated by the SageMaker Pipelines service, not by this class.
79+
See the Athena `StartQueryExecution
80+
<https://docs.aws.amazon.com/athena/latest/APIReference/API_StartQueryExecution.html>`_
81+
API for the meaning and allowed values of each field.
8482
"""
8583

84+
def __init__(
85+
self,
86+
query_string: str,
87+
database: Optional[Union[str, PipelineVariable]] = None,
88+
catalog: Optional[Union[str, PipelineVariable]] = None,
89+
work_group: Optional[Union[str, PipelineVariable]] = None,
90+
execution_parameters: Optional[List[Union[str, PipelineVariable]]] = None,
91+
output_location: Optional[Union[str, PipelineVariable]] = None,
92+
encryption_option: Optional[Union[str, PipelineVariable]] = None,
93+
kms_key: Optional[Union[str, PipelineVariable]] = None,
94+
expected_bucket_owner: Optional[Union[str, PipelineVariable]] = None,
95+
s3_acl_option: Optional[Union[str, PipelineVariable]] = None,
96+
result_reuse_max_age_in_minutes: Optional[Union[int, PipelineVariable]] = None,
97+
): # pylint: disable=too-many-positional-arguments
98+
"""Create a definition for an Athena query.
99+
100+
Args:
101+
query_string (str): The SQL query. Must be a literal string; bind runtime
102+
values with ``execution_parameters``.
103+
database (str, optional): The database the query runs in
104+
(``QueryExecutionContext.Database``).
105+
catalog (str, optional): The data catalog the query runs in
106+
(``QueryExecutionContext.Catalog``).
107+
work_group (str, optional): The Athena workgroup.
108+
execution_parameters (List[str], optional): Values for the ``?``
109+
placeholders in ``query_string``, in order.
110+
output_location (str, optional): S3 URI for the query results
111+
(``ResultConfiguration.OutputLocation``). May be omitted when the
112+
workgroup sets it.
113+
encryption_option (str, optional): ``SSE_S3``, ``SSE_KMS`` or ``CSE_KMS``
114+
(``ResultConfiguration.EncryptionConfiguration.EncryptionOption``).
115+
kms_key (str, optional): KMS key for ``SSE_KMS``/``CSE_KMS``
116+
(``ResultConfiguration.EncryptionConfiguration.KmsKey``).
117+
expected_bucket_owner (str, optional): Account ID expected to own the
118+
output bucket (``ResultConfiguration.ExpectedBucketOwner``).
119+
s3_acl_option (str, optional): Canned ACL for the results, e.g.
120+
``BUCKET_OWNER_FULL_CONTROL``
121+
(``ResultConfiguration.AclConfiguration.S3AclOption``).
122+
result_reuse_max_age_in_minutes (int, optional): When set, enables Athena
123+
query result reuse with this maximum age
124+
(``ResultReuseConfiguration.ResultReuseByAgeConfiguration``).
125+
"""
126+
self.query_string = query_string
127+
self.database = database
128+
self.catalog = catalog
129+
self.work_group = work_group
130+
self.execution_parameters = execution_parameters
131+
self.output_location = output_location
132+
self.encryption_option = encryption_option
133+
self.kms_key = kms_key
134+
self.expected_bucket_owner = expected_bucket_owner
135+
self.s3_acl_option = s3_acl_option
136+
self.result_reuse_max_age_in_minutes = result_reuse_max_age_in_minutes
137+
138+
def to_request(self) -> RequestType:
139+
"""Convert this config to an Athena ``StartQueryExecution`` request dict."""
140+
request = {"QueryString": self.query_string}
141+
142+
context = _drop_none({"Database": self.database, "Catalog": self.catalog})
143+
if context:
144+
request["QueryExecutionContext"] = context
145+
146+
if self.work_group is not None:
147+
request["WorkGroup"] = self.work_group
148+
if self.execution_parameters is not None:
149+
request["ExecutionParameters"] = list(self.execution_parameters)
150+
151+
encryption = _drop_none(
152+
{"EncryptionOption": self.encryption_option, "KmsKey": self.kms_key}
153+
)
154+
acl = _drop_none({"S3AclOption": self.s3_acl_option})
155+
result_configuration = _drop_none(
156+
{
157+
"OutputLocation": self.output_location,
158+
"EncryptionConfiguration": encryption or None,
159+
"ExpectedBucketOwner": self.expected_bucket_owner,
160+
"AclConfiguration": acl or None,
161+
}
162+
)
163+
if result_configuration:
164+
request["ResultConfiguration"] = result_configuration
165+
166+
if self.result_reuse_max_age_in_minutes is not None:
167+
request["ResultReuseConfiguration"] = {
168+
"ResultReuseByAgeConfiguration": {
169+
"Enabled": True,
170+
"MaxAgeInMinutes": self.result_reuse_max_age_in_minutes,
171+
}
172+
}
173+
174+
return request
175+
176+
177+
def _drop_none(values: Dict[str, Any]) -> Dict[str, Any]:
178+
return {key: value for key, value in values.items() if value is not None}
179+
180+
181+
ERR_STR_WITH_BOTH_QUERY_CONFIG_AND_ARGUMENTS = (
182+
"AthenaStep {step_name} cannot have both query_config and arguments."
183+
)
184+
185+
ERR_STR_WITHOUT_QUERY_CONFIG_AND_ARGUMENTS = (
186+
"AthenaStep {step_name} must have either query_config or arguments."
187+
)
188+
189+
190+
class AthenaStep(ConfigurableRetryStep):
191+
"""Runs an Amazon Athena query as part of a SageMaker Pipeline."""
192+
86193
def __init__(
87194
self,
88195
name: str,
89-
arguments: Dict[str, Any],
196+
query_config: Optional[AthenaQueryConfig] = None,
197+
arguments: Optional[Dict[str, Any]] = None,
90198
display_name: Optional[str] = None,
91199
description: Optional[str] = None,
92200
depends_on: Optional[List[Union[str, Step, StepCollection]]] = None,
93201
cache_config: Optional[CacheConfig] = None,
94202
retry_policies: Optional[List[RetryPolicy]] = None,
95-
):
203+
): # pylint: disable=too-many-positional-arguments
96204
"""Construct an ``AthenaStep``.
97205
206+
Exactly one of ``query_config`` and ``arguments`` must be given.
207+
98208
Args:
99209
name: The name of the step.
100-
arguments: The Athena ``StartQueryExecution`` request as a dict
101-
(PascalCase keys, e.g. ``QueryString``, ``WorkGroup``,
102-
``QueryExecutionContext``, ``ResultConfiguration``,
103-
``ExecutionParameters``). Values may be literals or pipeline
104-
variables, except ``QueryString`` which must be a literal.
210+
query_config: The Athena query to run.
211+
arguments: The full Athena ``StartQueryExecution`` request as a dict
212+
(PascalCase keys), emitted unchanged. Use it for request fields
213+
that ``AthenaQueryConfig`` does not cover.
105214
display_name: Optional display name.
106215
description: Optional description.
107216
depends_on: Optional explicit step dependencies. Wrap an entry in
@@ -118,9 +227,16 @@ def __init__(
118227
depends_on=depends_on,
119228
retry_policies=retry_policies,
120229
)
121-
# Defensive deep-copy so caller-side mutations after construction do
122-
# not leak into the compiled pipeline definition.
123-
self.args: Dict[str, Any] = copy.deepcopy(arguments)
230+
231+
if query_config is None and arguments is None:
232+
raise ValueError(ERR_STR_WITHOUT_QUERY_CONFIG_AND_ARGUMENTS.format(step_name=name))
233+
if query_config is not None and arguments is not None:
234+
raise ValueError(ERR_STR_WITH_BOTH_QUERY_CONFIG_AND_ARGUMENTS.format(step_name=name))
235+
236+
# Copy so caller-side mutations after construction do not leak into the
237+
# compiled pipeline definition.
238+
request = query_config.to_request() if query_config is not None else arguments
239+
self.args: Dict[str, Any] = copy.deepcopy(request)
124240
self.cache_config = cache_config
125241
self._properties = Properties(
126242
step_name=name,

‎sagemaker-mlops/tests/integ/workflow/test_athena_step.py‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
* a valid query drives the execution to ``Succeeded`` and the result CSV is
1818
written to the configured S3 output location,
1919
* an invalid query drives the execution to ``Failed`` with a ``FailureReason``,
20-
* a full-shape ``arguments`` dict, together with ``CacheConfig`` and
20+
* a full ``AthenaQueryConfig``, together with ``CacheConfig`` and
2121
``RetryPolicies``, is accepted by ``CreatePipeline`` and round-trips
2222
through ``DescribePipeline`` unchanged.
2323
@@ -40,7 +40,7 @@
4040

4141
from sagemaker.core.common_utils import unique_name_from_base
4242
from sagemaker.core.workflow.parameters import ParameterString
43-
from sagemaker.mlops.workflow.athena_step import AthenaStep
43+
from sagemaker.mlops.workflow.athena_step import AthenaQueryConfig, AthenaStep
4444
from sagemaker.mlops.workflow.pipeline import Pipeline
4545
from sagemaker.mlops.workflow.retry import StepExceptionTypeEnum, StepRetryPolicy
4646
from sagemaker.mlops.workflow.steps import CacheConfig
@@ -106,10 +106,10 @@ def test_athena_step_query_succeeds_and_writes_result(sagemaker_session_for_pipe
106106

107107
step = AthenaStep(
108108
name="AthenaQuery",
109-
arguments={
110-
"QueryString": "SELECT 1 AS test_result",
111-
"ResultConfiguration": {"OutputLocation": output_location_parameter},
112-
},
109+
query_config=AthenaQueryConfig(
110+
query_string="SELECT 1 AS test_result",
111+
output_location=output_location_parameter,
112+
),
113113
)
114114
pipeline = Pipeline(
115115
name=pipeline_name,
@@ -143,10 +143,10 @@ def test_athena_step_invalid_query_fails(sagemaker_session_for_pipeline, role):
143143

144144
step = AthenaStep(
145145
name="AthenaInvalidQuery",
146-
arguments={
147-
"QueryString": "SELCT INVALID SYNTAX FROM nonexistent_table_xyz",
148-
"ResultConfiguration": {"OutputLocation": output_location_parameter},
149-
},
146+
query_config=AthenaQueryConfig(
147+
query_string="SELCT INVALID SYNTAX FROM nonexistent_table_xyz",
148+
output_location=output_location_parameter,
149+
),
150150
)
151151
pipeline = Pipeline(
152152
name=pipeline_name,
@@ -165,23 +165,23 @@ def test_athena_step_invalid_query_fails(sagemaker_session_for_pipeline, role):
165165
_cleanup(pipeline, execution, sagemaker_session_for_pipeline, bucket, key_prefix)
166166

167167

168-
def test_athena_step_full_arguments_roundtrip(sagemaker_session_for_pipeline, role):
169-
"""Full-shape ``arguments`` plus ``CacheConfig``/``RetryPolicies`` are accepted at
168+
def test_athena_step_full_query_config_roundtrip(sagemaker_session_for_pipeline, role):
169+
"""A full ``AthenaQueryConfig`` plus ``CacheConfig``/``RetryPolicies`` are accepted at
170170
``CreatePipeline`` and come back unchanged from ``DescribePipeline``."""
171171
pipeline_name = unique_name_from_base("athena-step-integ")
172172
output_location_parameter = ParameterString(name=_OUTPUT_LOCATION_PARAMETER)
173173

174174
step = AthenaStep(
175175
name="AthenaFullQuery",
176-
arguments={
177-
"QueryString": "SELECT 1",
178-
"QueryExecutionContext": {"Database": "default", "Catalog": "AwsDataCatalog"},
179-
"ResultConfiguration": {
180-
"OutputLocation": output_location_parameter,
181-
"EncryptionConfiguration": {"EncryptionOption": "SSE_S3"},
182-
},
183-
"WorkGroup": "primary",
184-
},
176+
query_config=AthenaQueryConfig(
177+
query_string="SELECT 1",
178+
database="default",
179+
catalog="AwsDataCatalog",
180+
work_group="primary",
181+
output_location=output_location_parameter,
182+
encryption_option="SSE_S3",
183+
result_reuse_max_age_in_minutes=60,
184+
),
185185
cache_config=CacheConfig(enable_caching=True, expire_after="PT1H"),
186186
retry_policies=[
187187
StepRetryPolicy(

0 commit comments

Comments
 (0)