Skip to content

Commit 460efcc

Browse files
committed
feat: add inital deploy and notify teams for prefect including examples
1 parent c4cac9e commit 460efcc

3 files changed

Lines changed: 241 additions & 7 deletions

File tree

‎examples/prefect/deploy_flow.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
"""Example usage of deploy function"""
2+
3+
from datetime import timedelta
4+
from os import environ
5+
6+
from prefect import flow
7+
from prefect.artifacts import create_table_artifact
8+
9+
from osw.utils.prefect import DeployConfig, DeployParam, deploy, tagsStrToList
10+
11+
# Set environment variables
12+
environ["PREFECT_DEPLOYMENT_NAME"] = "osw-python-deploy-example"
13+
environ["PREFECT_DEPLOYMENT_DESCRIPTION"] = "Deployment of notify_teams.py"
14+
environ["PREFECT_DEPLOYMENT_VERSION"] = "0.0.1"
15+
environ["PREFECT_DEPLOYMENT_TAGS"] = "osw-python,example-deploy-flow"
16+
environ["PREFECT_DEPLOYMENT_INTERVAL_MIN"] = "1"
17+
# environ["PREFECT_DEPLOYMENT_CRON"] = "* * * * *"
18+
19+
20+
@flow(log_prints=True)
21+
def example_flow_to_deploy():
22+
"""Example flow to be deployed"""
23+
print(f"Execution of example: {example_flow_to_deploy.__name__}!")
24+
# set example table artifact
25+
create_table_artifact(
26+
key="example-table",
27+
table=[
28+
{"name": "Alice", "age": 24},
29+
{"name": "Bob", "age": 25},
30+
],
31+
description="Example table artifact",
32+
)
33+
34+
35+
if __name__ == "__main__":
36+
"""Deploy the example flow"""
37+
# Example using environment variables
38+
deploy(
39+
DeployParam(
40+
deployments=[
41+
DeployConfig(
42+
flow=example_flow_to_deploy,
43+
name=environ.get("PREFECT_DEPLOYMENT_NAME"),
44+
description=environ.get("PREFECT_DEPLOYMENT_DESCRIPTION"),
45+
version=environ.get("PREFECT_DEPLOYMENT_VERSION"),
46+
tags=tagsStrToList(environ.get("PREFECT_DEPLOYMENT_TAGS")),
47+
interval=timedelta(
48+
minutes=int(environ.get("PREFECT_DEPLOYMENT_INTERVAL_MIN"))
49+
), # either interval or cron
50+
# cron=environ.get("PREFECT_DEPLOYMENT_CRON"),
51+
)
52+
],
53+
# remove_existing_deployments=True,
54+
)
55+
)
56+
57+
# Clear secret environment variables
58+
del environ["PREFECT_DEPLOYMENT_NAME"]
59+
del environ["PREFECT_DEPLOYMENT_DESCRIPTION"]
60+
del environ["PREFECT_DEPLOYMENT_VERSION"]
61+
del environ["PREFECT_DEPLOYMENT_TAGS"]
62+
del environ["PREFECT_DEPLOYMENT_INTERVAL_MIN"]
63+
# del environ["PREFECT_DEPLOYMENT_CRON"]

‎examples/prefect/notify_teams.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
"""Example of sending notifications to MS Teams on prefect flow failures"""
2+
3+
from os import environ
4+
5+
from prefect import flow
6+
from pydantic import SecretStr
7+
8+
from osw.utils.prefect import NotifyTeams, NotifyTeamsParam
9+
10+
# Prerequisite: Set environment variable TEAMS_WEBHOOK_URL
11+
# in CLI: export TEAMS_WEBHOOK_URL="https://prod..."
12+
# in python uncomment below, DO NOT PUSH SECRETS TO GIT
13+
14+
# environ["TEAMS_WEBHOOK_URL"] = "https://prod..."
15+
16+
17+
# Decorator must be configured with on_failure argument
18+
@flow(
19+
# Microsoft Teams notification on failure ->
20+
# on_failure use `notify_teams` function without brackets as list element
21+
on_failure=[
22+
NotifyTeams(
23+
NotifyTeamsParam(
24+
teams_webhook_url=SecretStr(environ.get("TEAMS_WEBHOOK_URL")),
25+
# OPTIONAL, will be empty if no deploment is assigned
26+
deployment_name="osw-python-notify-teams-example",
27+
)
28+
).notify_teams
29+
],
30+
log_prints=True,
31+
)
32+
def example_error_flow():
33+
"""Test flow that always fails"""
34+
35+
raise ValueError(
36+
"oops! LOREM IPSUM DOLOR SIT AMET CONSECTETUR ADIPISICING ELIT " * 1
37+
)
38+
39+
40+
if __name__ == "__main__":
41+
example_error_flow()
42+
# Clear secret environment variable
43+
del environ["TEAMS_WEBHOOK_URL"]

‎src/osw/utils/prefect.py‎

Lines changed: 135 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,31 @@
11
"""Prefect utils as support for OpenSemanticWorld."""
22

3+
import asyncio
4+
import re
5+
import sys
6+
from datetime import timedelta
7+
from importlib.metadata import version
8+
from typing import Iterable, List, Optional
9+
10+
from packaging.specifiers import SpecifierSet
11+
from prefect import Flow, get_client, serve
312
from prefect.blocks.notifications import MicrosoftTeamsWebhook
413
from prefect.client.schemas.objects import FlowRun
514
from prefect.settings import PREFECT_API_URL
615
from prefect.states import State
716
from pydantic import BaseModel, SecretStr
817

18+
# from prefect.settings import PREFECT_API_URL
19+
920

21+
# ------------------------------ NOTIFICATIONS ------------------------------
1022
class NotifyTeamsParam(BaseModel):
1123
"""Parameter set for notifying Microsoft Teams using class NotifyTeams"""
1224

13-
deployment_name: str
1425
teams_webhook_url: SecretStr
26+
"""Microsoft Teams webhook URL containing a secret"""
27+
deployment_name: Optional[str] = None
28+
"""Deployment name to be displayed in the notification"""
1529

1630

1731
class NotifyTeams(NotifyTeamsParam):
@@ -29,17 +43,21 @@ def notify_teams(
2943

3044
host_url = str(PREFECT_API_URL.value()).replace("/api", "")
3145

46+
_flow_run = f"**🚨Flow Run: [{flow.name} > {flow_run.name}]({host_url}/flow-runs/flow-run/{flow_run.id}) ❗{state.name}❗**\n\n" # noqa
47+
3248
if flow_run.deployment_id is not None:
49+
# Assigned deployment found
3350
deployment_url = (
3451
f"{host_url}/deployments/deployment/{flow_run.deployment_id}"
3552
)
53+
if self.deployment_name == "" or self.deployment_name is None:
54+
_deployment = f"🚀 Deployment: _[{flow_run.deployment_id}]({deployment_url})_\n\n" # noqa
55+
else:
56+
_deployment = f"🚀 Deployment: _[{self.deployment_name}]({deployment_url})_\n\n" # noqa
3657
else:
37-
deployment_url = ""
38-
_flow_run = f"**🚨Flow Run: [{flow.name} > {flow_run.name}]({host_url}/flow-runs/flow-run/{flow_run.id}) ❗{state.name}❗**\n\n" # noqa
39-
if self.deployment_name == "" or self.deployment_name is None:
40-
_deployment = f"🚀 Deployment: _[{flow_run.deployment_id}]({deployment_url})_\n\n" # noqa
41-
else:
42-
_deployment = f"🚀 Deployment: _[{self.deployment_name}]({deployment_url})_\n\n" # noqa
58+
# No deployment assigned
59+
_deployment = "🚀 Deployment: _Just flow, no deployment_\n\n"
60+
4361
_ts = f"🕑 Timestamp: _{flow_run.state.timestamp.strftime('%Y-%m-%d %H:%M:%S %Z')}_\n\n" # noqa
4462
if flow_run.tags != []:
4563
_tags = f"🏷️ Tags: _#{' #'.join(flow_run.tags)}_\n\n"
@@ -48,6 +66,116 @@ def notify_teams(
4866

4967
_message = f"📜 Message:\n\n_`{state.message}`_"
5068

69+
# # DEBUG
70+
# print(_flow_run + _deployment + _ts + _tags + _message)
71+
# print(f"Teams webhook URL: {self.teams_webhook_url}")
72+
# print(f"Deployment name: {self.deployment_name}")
73+
# print(f"Flow name: {flow.name}")
74+
# print(f"Flow run name: {flow_run.name}")
75+
# print(f"Flow run ID: {flow_run.id}")
76+
5177
MicrosoftTeamsWebhook(url=self.teams_webhook_url.get_secret_value()).notify(
5278
body=(_flow_run + _deployment + _ts + _tags + _message)
5379
)
80+
81+
82+
# ------------------------------- DEPLOYMENTS -------------------------------
83+
def tagsStrToList(tags: str) -> List[str]:
84+
"""Remove tags whitespaces, newlines, tabs, empty strings, split comma"""
85+
return list(filter(None, re.sub(r"\s+", "", tags).split(",")))
86+
87+
88+
class DeployConfig(BaseModel):
89+
"""Prefect deployment configuration"""
90+
91+
flow: Flow
92+
name: str | None = None
93+
description: str | None = None
94+
interval: Iterable[int | float | timedelta] | int | float | timedelta | None = None
95+
cron: Iterable[str] | str | None = None
96+
version: str | None = None
97+
tags: List[str] | None = None
98+
99+
# Parameters that could be added in future, see to_deployment function:
100+
# rrule: Iterable[str] | str | None = None
101+
# paused: bool | None = None
102+
# schedules: List[FlexibleScheduleList] | None = None
103+
# schedule: SCHEDULE_TYPES | None = None
104+
# is_schedule_active: bool | None = None
105+
# parameters: dict | None = None
106+
# triggers: List[DeploymentTriggerTypes | TriggerTypes] | None = None
107+
# enforce_parameter_schema: bool = False
108+
# work_pool_name: str | None = None
109+
# work_queue_name: str | None = None
110+
# job_variables: Dict[str, Any] | None = None
111+
# deployment_id: str | None = None
112+
# prefect_api_url: str = PREFECT_API_URL
113+
class Config:
114+
arbitrary_types_allowed = True
115+
116+
117+
class DeployParam(BaseModel):
118+
"""Parameter set for deploying flows as deployments"""
119+
120+
deployments: List[DeployConfig]
121+
"""List of deployments to be served"""
122+
# TODO: Implement remove_existing_deployments
123+
remove_existing_deployments: Optional[bool] = False
124+
"""Will remove existing deployments of the specified flows/software"""
125+
# TODO: Add parameter for OSW support in next version
126+
127+
128+
async def _deploy(param: DeployParam):
129+
"""programmatic deployment supported in newer prefect versions
130+
This should become part of osw-python
131+
"""
132+
133+
deployments = []
134+
135+
for deploy_config in param.deployments:
136+
flow: Flow = deploy_config.flow
137+
# Set deployment name if not provided
138+
if deploy_config.name is None or deploy_config.name == "":
139+
deploy_config.name = flow.name + "-deployment"
140+
config = await flow.to_deployment(
141+
name=deploy_config.name,
142+
tags=deploy_config.tags,
143+
cron=deploy_config.cron,
144+
interval=deploy_config.interval,
145+
description=deploy_config.description,
146+
version=deploy_config.version,
147+
)
148+
await config.apply() # returns the deployment_uuid
149+
150+
deployments.append(config)
151+
152+
# fetch flow uuid
153+
async with get_client() as client:
154+
response = await client.read_flow_by_name(flow.name)
155+
print(response.json())
156+
flow_uuid = response.id
157+
print("Flow UUID:", flow_uuid)
158+
159+
# prefect_domain = (
160+
# environ.get("PREFECT_API_URL").split("//")[-1].split("/")[0]
161+
# ) # noqa
162+
# print("Prefect domain:", prefect_domain)
163+
# start agent to serve deployment
164+
# await deploy_config.flow.serve(name=deployment_name)
165+
if version("prefect") in SpecifierSet(">=3.0"):
166+
return deployments
167+
else:
168+
await serve(*deployments)
169+
170+
171+
def deploy(param: DeployParam):
172+
"""Function to serve configured flows as deployments by python version."""
173+
if sys.version_info >= (3, 11):
174+
# python >= 3.11
175+
with asyncio.Runner() as runner:
176+
deployments = runner.run(_deploy(param))
177+
else:
178+
# python < 3.11
179+
deployments = asyncio.run(_deploy(param))
180+
if version("prefect") in SpecifierSet(">=3.0"):
181+
serve(*deployments)

0 commit comments

Comments
 (0)