Skip to content

Commit 9381dc3

Browse files
committed
add add support for python 3.8 pydantic syntax and tests for prefect workflows
1 parent 1ab7e05 commit 9381dc3

4 files changed

Lines changed: 185 additions & 80 deletions

File tree

‎examples/prefect/deploy_flow.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from prefect import flow
77
from prefect.artifacts import create_table_artifact
88

9-
from osw.utils.workflow import DeployConfig, DeployParam, deploy, tagsStrToList
9+
from osw.utils.workflow import DeployConfig, DeployParam, deploy, tags_str_to_list
1010

1111
# Set environment variables
1212
environ["PREFECT_DEPLOYMENT_NAME"] = "osw-python-deploy-example"
@@ -43,11 +43,9 @@ def example_flow_to_deploy():
4343
name=environ.get("PREFECT_DEPLOYMENT_NAME"),
4444
description=environ.get("PREFECT_DEPLOYMENT_DESCRIPTION"),
4545
version=environ.get("PREFECT_DEPLOYMENT_VERSION"),
46-
tags=tagsStrToList(environ.get("PREFECT_DEPLOYMENT_TAGS")),
46+
tags=tags_str_to_list(environ.get("PREFECT_DEPLOYMENT_TAGS")),
4747
interval=timedelta(
48-
minutes=int(
49-
environ.get("PREFECT_DEPLOYMENT_INTERVAL_MIN")
50-
)
48+
minutes=int(environ.get("PREFECT_DEPLOYMENT_INTERVAL_MIN"))
5149
), # either interval or cron
5250
# cron=environ.get("PREFECT_DEPLOYMENT_CRON"),
5351
)

‎setup.cfg‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ dev =
111111

112112
# Add here test requirements (semicolon/line-separated)
113113
testing =
114+
pytest-asyncio
115+
prefect
114116
setuptools
115117
pytest
116118
pytest-cov
@@ -147,6 +149,8 @@ norecursedirs =
147149
build
148150
.tox
149151
testpaths = tests
152+
asyncio_default_fixture_loop_scope = function
153+
150154
# Use pytest markers to select/deselect specific tests
151155
# markers =
152156
# slow: mark tests as slow (deselect with '-m "not slow"')

‎src/osw/utils/workflow.py‎

Lines changed: 84 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,17 @@
55
import sys
66
from datetime import timedelta
77
from importlib.metadata import version
8-
from typing import Iterable, List, Optional
8+
from inspect import signature
9+
from typing import Any, Dict, Iterable, List, Optional, Union
10+
911
from packaging.specifiers import SpecifierSet
1012
from prefect import Flow, serve
1113
from prefect.blocks.notifications import MicrosoftTeamsWebhook
1214
from prefect.client.schemas.objects import FlowRun
1315
from prefect.settings import PREFECT_API_URL
1416
from prefect.states import State
15-
from pydantic import BaseModel, SecretStr
16-
17-
# ------------------------------ TEST ------------------------------
18-
19-
20-
from prefect import flow
21-
22-
23-
@flow
24-
def example_flow_to_deploy():
25-
"""Example flow to be deployed"""
26-
print(f"Execution of example: {example_flow_to_deploy.__name__}!")
17+
from pydantic import SecretStr
18+
from pydantic.v1 import BaseModel
2719

2820

2921
# ------------------------------ NOTIFICATIONS ---------------------
@@ -35,12 +27,17 @@ class NotifyTeamsParam(BaseModel):
3527
deployment_name: Optional[str] = None
3628
"""Deployment name to be displayed in the notification"""
3729

30+
# allow arbitrary types for compatibility with pydantic v1
31+
class Config:
32+
arbitrary_types_allowed = True
33+
3834

3935
class NotifyTeams(NotifyTeamsParam):
4036
"""Notify Microsoft Teams channel using a webhook"""
4137

4238
def __init__(self, notify_teams_param: NotifyTeamsParam):
43-
super().__init__(**notify_teams_param.model_dump())
39+
# super().__init__(**notify_teams_param.model_dump()) # pydantic v2
40+
super().__init__(**notify_teams_param.dict()) # pydantic v1
4441

4542
def notify_teams(
4643
self,
@@ -72,54 +69,62 @@ def notify_teams(
7269
else:
7370
_tags = ""
7471

75-
_message = f"📜 Message:\n\n_`{state.message}`_"
76-
77-
# # DEBUG
78-
# print(_flow_run + _deployment + _ts + _tags + _message)
79-
# print(f"Teams webhook URL: {self.teams_webhook_url}")
80-
# print(f"Deployment name: {self.deployment_name}")
81-
# print(f"Flow name: {flow.name}")
82-
# print(f"Flow run name: {flow_run.name}")
83-
# print(f"Flow run ID: {flow_run.id}")
72+
if state.message is None:
73+
_message = "No message provided."
74+
else:
75+
_message = f"📜 Message:\n\n_`{state.message}`_"
8476

8577
MicrosoftTeamsWebhook(
86-
url=self.teams_webhook_url.get_secret_value()
78+
url=str(self.teams_webhook_url.get_secret_value())
8779
).notify(body=(_flow_run + _deployment + _ts + _tags + _message))
8880

8981

9082
# ------------------------------- DEPLOYMENTS -------------------------------
91-
def tagsStrToList(tags: str) -> List[str]:
83+
def tags_str_to_list(tags: str) -> List[str]:
9284
"""Remove tags whitespaces, newlines, tabs, empty strings, split comma"""
9385
return list(filter(None, re.sub(r"\s+", "", tags).split(",")))
9486

9587

88+
# def filter_arguments(func, args_dict):
89+
# """Filter arguments for a function based on its signature"""
90+
# sig = signature(func)
91+
# valid_params = sig.parameters
92+
# filtered_args = {k: v for k, v in args_dict.items() if k in valid_params}
93+
# return filtered_args
94+
95+
96+
def match_func_model_args(func, model: BaseModel) -> dict:
97+
"""Match function arguments with model attributes"""
98+
valid_params = set(signature(func).parameters)
99+
# model_attrs = model.model_dump().items() # pydantic v2
100+
model_attrs = model.dict().items() # pydantic v1
101+
matched_args = {k: v for k, v in model_attrs if k in valid_params}
102+
return matched_args
103+
104+
96105
class DeployConfig(BaseModel):
97106
"""Prefect deployment configuration"""
98107

99108
flow: Flow # to be excluded in `flow.to_deployment()` function
100-
name: str | None = None
101-
description: str | None = None
102-
interval: (
103-
Iterable[int | float | timedelta] | int | float | timedelta | None
104-
) = None
105-
cron: Iterable[str] | str | None = None
106-
version: str | None = None
107-
tags: List[str] | None = None
108-
109-
# Parameters that could be added in future, see to_deployment function:
110-
# rrule: Iterable[str] | str | None = None
111-
# paused: bool | None = None
112-
# schedules: List[FlexibleScheduleList] | None = None
113-
# schedule: SCHEDULE_TYPES | None = None
114-
# is_schedule_active: bool | None = None
115-
# parameters: dict | None = None
116-
# triggers: List[DeploymentTriggerTypes | TriggerTypes] | None = None
117-
# enforce_parameter_schema: bool = False
118-
# work_pool_name: str | None = None
119-
# work_queue_name: str | None = None
120-
# job_variables: Dict[str, Any] | None = None
121-
# deployment_id: str | None = None
122-
# prefect_api_url: str = PREFECT_API_URL
109+
# Union instead of | for compatibility with pydantic v1, python < 3.10
110+
name: Union[str, None] = None
111+
description: Union[str, None] = None
112+
interval: Union[
113+
Iterable[Union[int, float, timedelta]], int, float, timedelta, None
114+
] = None
115+
cron: Union[Iterable[str], str, None] = None
116+
version: Union[str, None] = None
117+
tags: Union[List[str], None] = None
118+
rrule: Union[Iterable[str], str, None] = None
119+
paused: Union[bool, None] = None
120+
is_schedule_active: Union[bool, None] = None
121+
parameters: Union[dict, None] = None
122+
enforce_parameter_schema: bool = False
123+
work_pool_name: Union[str, None] = None
124+
work_queue_name: Union[str, None] = None
125+
job_variables: Union[Dict[str, Any], None] = None
126+
deployment_id: Union[str, None] = None
127+
123128
class Config:
124129
arbitrary_types_allowed = True
125130

@@ -147,23 +152,19 @@ async def _deploy(param: DeployParam):
147152
# Set deployment name if not provided
148153
if deploy_config.name is None or deploy_config.name == "":
149154
deploy_config.name = flow.name + "-deployment"
150-
config = await flow.to_deployment(
151-
# Entpacken und ungleiche Parameter exkludieren (ggf. ext funktion schreiben mit inspect.signature -> fkt + dict input -> dict mit keys der args output)
152-
name=deploy_config.name,
153-
tags=deploy_config.tags,
154-
cron=deploy_config.cron,
155-
interval=deploy_config.interval,
156-
description=deploy_config.description,
157-
version=deploy_config.version,
158-
)
155+
156+
# Match valid args of flow.to_deployment and deploy_config
157+
kwargs = match_func_model_args(func=flow.to_deployment, model=deploy_config)
158+
# Set config via matching flow.to_deployment arguments
159+
config = await flow.to_deployment(**kwargs)
159160
await config.apply() # returns the deployment_uuid
160161

161162
deployments.append(config)
162163

163164
if version("prefect") in SpecifierSet(">=3.0"):
164165
print(f"prefect version IF: {version('prefect')}")
165166
# return deployments
166-
serve(*deployments)
167+
await serve(*deployments)
167168
else:
168169
print(f"prefect version ELSE: {version('prefect')}")
169170
await serve(*deployments)
@@ -182,20 +183,28 @@ def deploy(param: DeployParam):
182183
asyncio.run(_deploy(param))
183184

184185

185-
if __name__ == "__main__":
186-
187-
deploy(
188-
DeployParam(
189-
deployments=[
190-
DeployConfig(
191-
flow=example_flow_to_deploy,
192-
name="osw-python-deploy-example",
193-
description="Deployment of notify_teams.py",
194-
version="0.0.1",
195-
tags=["osw-python", "example-deploy-flow"],
196-
interval=timedelta(seconds=20),
197-
)
198-
],
199-
# remove_existing_deployments=True,
200-
)
201-
)
186+
# # ------------------------------- TEST -------------------------------
187+
# from prefect import flow
188+
189+
190+
# @flow
191+
# def osw_python_test_flow_to_deploy():
192+
# """Example flow to be deployed"""
193+
# print(f"Execution of example: {osw_python_test_flow_to_deploy.__name__}!")
194+
195+
196+
# if __name__ == "__main__":
197+
# deploy(
198+
# DeployParam(
199+
# deployments=[
200+
# DeployConfig(
201+
# flow=osw_python_test_flow_to_deploy,
202+
# name="osw-python-deployment-test",
203+
# description="Deployment of osw-python test flow",
204+
# version="0.0.1",
205+
# tags=["osw-python", "example-deploy-flow"],
206+
# )
207+
# ],
208+
# # remove_existing_deployments=True,
209+
# )
210+
# )

‎tests/utils/workflow.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
"""Test workflows for `osw-python` package"""
2+
3+
import subprocess
4+
from os import environ
5+
6+
import pytest
7+
from prefect import flow
8+
from prefect.testing.utilities import prefect_test_harness
9+
from pydantic import SecretStr
10+
11+
from osw.utils.workflow import (
12+
DeployConfig,
13+
DeployParam,
14+
NotifyTeams,
15+
NotifyTeamsParam,
16+
_deploy,
17+
deploy,
18+
tags_str_to_list,
19+
)
20+
21+
22+
# ------------------------------ NOTIFICATIONS ---------------------
23+
@flow(
24+
# Microsoft Teams notification on completion for testing
25+
# Notification only if env var
26+
on_completion=[
27+
NotifyTeams(
28+
NotifyTeamsParam(
29+
teams_webhook_url=SecretStr(environ.get("TEAMS_WEBHOOK_URL")),
30+
# OPTIONAL, will be empty if no deploment is assigned
31+
deployment_name="osw-python-notify-teams-test",
32+
)
33+
).notify_teams
34+
],
35+
log_prints=True,
36+
)
37+
@flow
38+
def osw_python_teams_notify_test_flow():
39+
"""Notify Microsoft Teams channel using a webhook"""
40+
return 42
41+
42+
43+
def test_notify_teams():
44+
"""Test of flow to notify Microsoft Teams channel using a webhook"""
45+
with prefect_test_harness():
46+
test_flow_run = osw_python_teams_notify_test_flow()
47+
assert test_flow_run == 42
48+
49+
50+
# ------------------------------- DEPLOYMENTS -------------------------------
51+
def test_tags_str_to_list(tags="osw-python,example-deploy-flow"):
52+
"""Test of conversion of tags string to list"""
53+
assert tags_str_to_list("osw-python,example-deploy-flow") == [
54+
"osw-python",
55+
"example-deploy-flow",
56+
]
57+
58+
59+
@flow
60+
def osw_python_test_flow_to_deploy():
61+
"""Example flow to be deployed"""
62+
print(f"Execution of example: {osw_python_test_flow_to_deploy.__name__}!")
63+
64+
65+
deploy_param = DeployParam(
66+
deployments=[
67+
DeployConfig(
68+
flow=osw_python_test_flow_to_deploy,
69+
name="osw-python-deployment-test",
70+
description="Deployment of osw-python test flow",
71+
version="0.0.1",
72+
tags=["osw-python", "example-deploy-flow"],
73+
)
74+
],
75+
# remove_existing_deployments=True,
76+
)
77+
78+
79+
@pytest.mark.asyncio
80+
async def test_deploy_serve():
81+
"""Test of deployment of example flow"""
82+
with prefect_test_harness():
83+
_deploy(param=deploy_param)
84+
85+
86+
@pytest.mark.skip(reason="Not deployable in test environment")
87+
def test_deploy_runner():
88+
"""Test of deployment of example flow"""
89+
with prefect_test_harness():
90+
# test environment shell:
91+
command = "prefect config set PREFECT_API_URL=http://127.0.0.1:8443/api"
92+
subprocess.run(command, shell=True)
93+
# Not deployable in local test environment
94+
deploy(param=deploy_param)

0 commit comments

Comments
 (0)