Skip to content

Commit bf9c58b

Browse files
dansoladansola
authored
Merge branch 'main' into guard-sync-task-in-async-context
2 parents 2cd8df0 + cbc2251 commit bf9c58b

66 files changed

Lines changed: 5659 additions & 722 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ dev-rs-dist:
113113
cli-docs-gen: ## Generate CLI documentation
114114
@echo "📖 Generating CLI documentation..."
115115
@uv run flyte gen docs --type markdown
116+
@uv run flyte gen docs --type json | uv run python maint_tools/check_cli_json.py
116117

117118
.PHONY: check-docstrings
118119
check-docstrings: ## Reject reStructuredText and NumPy sections in docstrings

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,12 @@ Run on the devbox:
177177
flyte run flyte_intro.py main --data '[1,2,3]'
178178
```
179179

180+
Check on the devbox — run state, UI and image registry endpoints, and the container image in use:
181+
182+
```bash
183+
flyte get devbox
184+
```
185+
180186
<img src="static/flyte-start-devbox.png" alt="Flyte Start Devbox">
181187

182188
<img src="static/flyte-hello-world.gif" alt="Flyte Hello World">

‎examples/genai/nemotron_omni_voice/README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,10 @@ print(r.json()["choices"][0]["message"]["content"])
104104
## Notes / knobs
105105

106106
- **Version pins are deliberate.** The Omni checkpoint needs **vLLM ≥ 0.20** and
107-
`--trust-remote-code`; both are set in `serve.py`. Bump `VLLM_VERSION` there if
108-
NVIDIA's recipe moves.
107+
`--trust-remote-code`. The vLLM pin comes from the plugin's `DEFAULT_VLLM_IMAGE`
108+
(which is well past that floor) and `--trust-remote-code` is set in `serve.py`.
109+
If NVIDIA's recipe moves to a newer vLLM, append a `.with_pip_packages("vllm==<version>")`
110+
layer to the image there.
109111
- **Want lower cost over lower latency?** Set `Scaling(replicas=(0, 1), scaledown_after=300)`
110112
to scale to zero when idle — at the cost of a cold start (multi-GB load) on the next request.
111113
- **BF16 instead of FP8** needs ~64 GB → switch `MODEL_REPO` to the BF16 variant

‎examples/genai/nemotron_omni_voice/serve.py‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
python examples/genai/nemotron_omni_voice/voice_client.py --endpoint <app-url>
4141
"""
4242

43-
from flyteplugins.vllm import VLLMAppEnvironment
43+
from flyteplugins.vllm import DEFAULT_VLLM_IMAGE, VLLMAppEnvironment
4444

4545
import flyte
4646
import flyte.app
@@ -60,21 +60,17 @@
6060
# ---------------------------------------------------------------------------
6161
# Image
6262
#
63-
# The Omni model needs vLLM >= 0.20 and trust-remote-code. We mirror the
64-
# plugin's default image (flashinfer for fast FP8 attention on Ada/L40S) but
65-
# pin a newer vLLM and add the audio decoding libraries vLLM uses to read the
66-
# incoming wav/mp3 payloads.
63+
# The Omni model needs vLLM >= 0.20 and trust-remote-code. The plugin's default
64+
# image already satisfies the version floor (and carries a flashinfer build
65+
# matching that vLLM, for fast FP8 attention on Ada/L40S), so this only adds the
66+
# audio decoding libraries vLLM uses to read the incoming wav/mp3 payloads.
67+
#
68+
# To follow a newer vLLM than the plugin pins, append your own layer:
69+
# .with_pip_packages("vllm==<version>")
6770
# ---------------------------------------------------------------------------
6871

69-
VLLM_VERSION = "0.20.0"
70-
7172
image = (
72-
flyte.Image.from_debian_base(name="nemotron-omni-vllm", install_flyte=False)
73-
.with_pip_packages("flashinfer-python", "flashinfer-cubin")
74-
.with_pip_packages("flashinfer-jit-cache", index_url="https://flashinfer.ai/whl/cu129")
75-
.with_pip_packages("flyteplugins-vllm", pre=True)
76-
# vLLM goes in its own layer (dependency conflict with flyte on protovalidate).
77-
.with_pip_packages(f"vllm=={VLLM_VERSION}", "transformers>=4.57.0")
73+
DEFAULT_VLLM_IMAGE.clone(name="nemotron-omni-vllm")
7874
# Audio decoding for the multimodal input pipeline.
7975
.with_pip_packages("librosa", "soundfile")
8076
)

‎examples/genai/sglang/sglang_app.py‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -39,29 +39,15 @@
3939
```
4040
"""
4141

42-
from flyteplugins.sglang import SGLangAppEnvironment
42+
from flyteplugins.sglang import DEFAULT_SGLANG_IMAGE, SGLangAppEnvironment
4343

4444
import flyte
4545
import flyte.app
4646

47-
image = (
48-
flyte.Image.from_debian_base(name="sglang-app-image", install_flyte=False)
49-
.with_apt_packages("libnuma-dev", "wget", "curl", "openssl", "pkg-config", "libssl-dev", "build-essential")
50-
.with_commands(
51-
[
52-
"wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb",
53-
"dpkg -i cuda-keyring_1.1-1_all.deb",
54-
"apt-get update",
55-
"apt-get install -y cuda-toolkit-12-8",
56-
]
57-
)
58-
.with_commands(["curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && . $HOME/.cargo/env"])
59-
.with_env_vars({"CUDA_HOME": "/usr/local/cuda-12.8", "PATH": "/root/.cargo/bin:/usr/local/cuda-12.8/bin:$PATH"})
60-
.with_pip_packages("flashinfer-python", "flashinfer-cubin")
61-
.with_pip_packages("flashinfer-jit-cache", index_url="https://flashinfer.ai/whl/cu128")
62-
.with_pip_packages("sglang==0.5.7")
63-
.with_pip_packages("flyteplugins-sglang")
64-
)
47+
# The plugin's default image: an SGLang the model loader supports, plus the CUDA toolkit
48+
# matching the CUDA major that SGLang's own wheels pin. Extend it with
49+
# `.clone(name=...).with_pip_packages(...)` when an app needs extra dependencies.
50+
image = DEFAULT_SGLANG_IMAGE
6551

6652
# Define the SGLang app environment for the smallest Qwen3 model
6753
sglang_app = SGLangAppEnvironment(

‎examples/genai/sglang/sglang_app_artifact.py‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@
7373
```
7474
"""
7575

76-
from flyteplugins.sglang import SGLangAppEnvironment
76+
from flyteplugins.sglang import DEFAULT_SGLANG_IMAGE, SGLangAppEnvironment
7777

7878
import flyte
7979
import flyte.app
@@ -84,24 +84,10 @@
8484
# tail, with '.' replaced by '-'. Pass `artifact_name=` to hf_model to override it.
8585
ARTIFACT_NAME = "SmolLM2-135M-Instruct"
8686

87-
image = (
88-
flyte.Image.from_debian_base(name="sglang-app-image", install_flyte=False)
89-
.with_apt_packages("libnuma-dev", "wget", "curl", "openssl", "pkg-config", "libssl-dev", "build-essential")
90-
.with_commands(
91-
[
92-
"wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb",
93-
"dpkg -i cuda-keyring_1.1-1_all.deb",
94-
"apt-get update",
95-
"apt-get install -y cuda-toolkit-12-8",
96-
]
97-
)
98-
.with_commands(["curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && . $HOME/.cargo/env"])
99-
.with_env_vars({"CUDA_HOME": "/usr/local/cuda-12.8", "PATH": "/root/.cargo/bin:/usr/local/cuda-12.8/bin:$PATH"})
100-
.with_pip_packages("flashinfer-python", "flashinfer-cubin")
101-
.with_pip_packages("flashinfer-jit-cache", index_url="https://flashinfer.ai/whl/cu128")
102-
.with_pip_packages("sglang==0.5.7")
103-
.with_pip_packages("flyteplugins-sglang")
104-
)
87+
# The plugin's default image: an SGLang the model loader supports, plus the CUDA toolkit
88+
# matching the CUDA major that SGLang's own wheels pin. Extend it with
89+
# `.clone(name=...).with_pip_packages(...)` when an app needs extra dependencies.
90+
image = DEFAULT_SGLANG_IMAGE
10591

10692
smollm2_app = SGLangAppEnvironment(
10793
name="smollm2-135m-sglang",

‎examples/genai/sglang/sglang_app_sharded.py‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -39,29 +39,15 @@
3939
```
4040
"""
4141

42-
from flyteplugins.sglang import SGLangAppEnvironment
42+
from flyteplugins.sglang import DEFAULT_SGLANG_IMAGE, SGLangAppEnvironment
4343

4444
import flyte
4545
import flyte.app
4646

47-
image = (
48-
flyte.Image.from_debian_base(name="sglang-app-image", install_flyte=False)
49-
.with_apt_packages("libnuma-dev", "wget", "curl", "openssl", "pkg-config", "libssl-dev", "build-essential")
50-
.with_commands(
51-
[
52-
"wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb",
53-
"dpkg -i cuda-keyring_1.1-1_all.deb",
54-
"apt-get update",
55-
"apt-get install -y cuda-toolkit-12-8",
56-
]
57-
)
58-
.with_commands(["curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && . $HOME/.cargo/env"])
59-
.with_env_vars({"CUDA_HOME": "/usr/local/cuda-12.8", "PATH": "/root/.cargo/bin:/usr/local/cuda-12.8/bin:$PATH"})
60-
.with_pip_packages("flashinfer-python", "flashinfer-cubin")
61-
.with_pip_packages("flashinfer-jit-cache", index_url="https://flashinfer.ai/whl/cu128")
62-
.with_pip_packages("sglang==0.5.7")
63-
.with_pip_packages("flyteplugins-sglang")
64-
)
47+
# The plugin's default image: an SGLang the model loader supports, plus the CUDA toolkit
48+
# matching the CUDA major that SGLang's own wheels pin. Extend it with
49+
# `.clone(name=...).with_pip_packages(...)` when an app needs extra dependencies.
50+
image = DEFAULT_SGLANG_IMAGE
6551

6652
# Define the SGLang app environment for the smallest Qwen3 model
6753
sglang_app = SGLangAppEnvironment(

‎examples/genai/vllm/vllm_app.py‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
```
4040
"""
4141

42-
from flyteplugins.vllm import VLLMAppEnvironment
42+
from flyteplugins.vllm import DEFAULT_VLLM_IMAGE, VLLMAppEnvironment
4343

4444
import flyte
4545
import flyte.app
@@ -50,16 +50,11 @@
5050
model_hf_path="Qwen/Qwen3-0.6B",
5151
model_id="qwen3-0.6b",
5252
resources=flyte.Resources(cpu="4", memory="16Gi", gpu="L40s:1", disk="10Gi"),
53-
image=(
54-
flyte.Image.from_debian_base(
55-
name="vllm-app-image",
56-
install_flyte=False,
57-
)
58-
.with_pip_packages("flashinfer-python", "flashinfer-cubin")
59-
.with_pip_packages("flashinfer-jit-cache", index_url="https://flashinfer.ai/whl/cu129")
60-
.with_pip_packages("vllm==0.11.0", "transformers==4.57.6")
61-
.with_pip_packages("flyteplugins-vllm")
62-
),
53+
# `image` defaults to DEFAULT_VLLM_IMAGE, which pins a vLLM the plugin's model loader
54+
# supports along with a matching flashinfer build. It is named here only to make that
55+
# explicit; extend it with `.clone(name=...).with_pip_packages(...)` when an app needs
56+
# extra dependencies.
57+
image=DEFAULT_VLLM_IMAGE,
6358
stream_model=True, # Stream model directly from blob store to GPU
6459
scaling=flyte.app.Scaling(
6560
replicas=(0, 1), # (min_replicas, max_replicas)

‎examples/genai/vllm/vllm_app_artifact.py‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@
6868
```
6969
"""
7070

71-
from flyteplugins.vllm import VLLMAppEnvironment
71+
from flyteplugins.vllm import DEFAULT_VLLM_IMAGE, VLLMAppEnvironment
7272

7373
import flyte
7474
import flyte.app
@@ -79,18 +79,9 @@
7979
# tail, with '.' replaced by '-'. Pass `artifact_name=` to hf_model to override it.
8080
ARTIFACT_NAME = "SmolLM2-135M-Instruct"
8181

82-
image = (
83-
flyte.Image.from_debian_base(
84-
name="vllm-app-image",
85-
install_flyte=False,
86-
)
87-
.with_pip_packages("flashinfer-python", "flashinfer-cubin")
88-
.with_pip_packages("flashinfer-jit-cache", index_url="https://flashinfer.ai/whl/cu129")
89-
# transformers is pinned deliberately: newer releases break the tokenizer load
90-
# path used by the plugin's fserve entrypoint.
91-
.with_pip_packages("vllm==0.11.0", "transformers==4.57.6")
92-
.with_pip_packages("flyteplugins-vllm")
93-
)
82+
# The plugin's default image: a vLLM the model loader supports, a flashinfer build matching
83+
# that vLLM's CUDA major, and transformers left to vLLM's own floor (it pins one).
84+
image = DEFAULT_VLLM_IMAGE
9485

9586
smollm2_app = VLLMAppEnvironment(
9687
name="smollm2-135m-vllm",

‎examples/genai/vllm/vllm_app_sharded.py‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
```
4040
"""
4141

42-
from flyteplugins.vllm import VLLMAppEnvironment
42+
from flyteplugins.vllm import DEFAULT_VLLM_IMAGE, VLLMAppEnvironment
4343

4444
import flyte
4545
import flyte.app
@@ -50,13 +50,9 @@
5050
model_hf_path="Qwen/Qwen3-14B",
5151
model_id="qwen3-14b",
5252
resources=flyte.Resources(cpu="36", memory="300Gi", gpu="L40s:4", disk="300Gi", shm="auto"),
53-
image=(
54-
flyte.Image.from_debian_base(name="vllm-app-image", install_flyte=False)
55-
.with_pip_packages("flashinfer-python", "flashinfer-cubin")
56-
.with_pip_packages("flashinfer-jit-cache", index_url="https://flashinfer.ai/whl/cu129")
57-
.with_pip_packages("vllm==0.11.0")
58-
.with_pip_packages("flyteplugins-vllm")
59-
),
53+
# The plugin's default image already carries a vLLM its sharded loader supports, plus a
54+
# matching flashinfer build, so there is nothing to add for tensor-parallel serving.
55+
image=DEFAULT_VLLM_IMAGE,
6056
stream_model=True, # Stream model directly from blob store to GPU
6157
scaling=flyte.app.Scaling(
6258
replicas=(0, 1), # (min_replicas, max_replicas)

0 commit comments

Comments
 (0)