v2.6.5 #377
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish | |
| on: | |
| release: | |
| types: [published] | |
| workflow_dispatch: | |
| inputs: | |
| tag: | |
| description: "Release tag to rebuild, e.g. v2.6.0. Rebuilds and pushes the images for an already-published release; every PyPI upload step is release-gated, so nothing is re-published." | |
| required: true | |
| pull_request: | |
| paths: | |
| - ".github/workflows/publish.yml" | |
| - "maint_tools/build_default_image.py" | |
| env: | |
| # Release runs use the tag they fired on; manual reruns use the tag input. | |
| RELEASE_TAG: ${{ inputs.tag || github.ref_name }} | |
| jobs: | |
| rs-controller-wheels: | |
| name: Build RS controller wheel (${{ matrix.target }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| # one flaky target shouldn't cancel the wheels that already built | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - target: x86_64 | |
| os: ubuntu-latest | |
| - target: aarch64 | |
| os: ubuntu-24.04-arm | |
| - target: aarch64-apple-darwin | |
| os: macos-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Set version from tag | |
| run: | | |
| if [[ "$RELEASE_TAG" == v* ]]; then | |
| VERSION="${RELEASE_TAG#v}" | |
| else | |
| VERSION="0.0.0.dev0" | |
| fi | |
| echo "VERSION=$VERSION" >> $GITHUB_ENV | |
| sed "s/^version = .*/version = \"$VERSION\"/" rs_controller/pyproject.toml > tmp && mv tmp rs_controller/pyproject.toml | |
| echo "Set version to $VERSION" | |
| cat rs_controller/pyproject.toml | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: auto | |
| args: --release --out dist -m rs_controller/Cargo.toml | |
| sccache: true | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: rs-controller-wheel-${{ matrix.target }} | |
| path: dist/*.whl | |
| rs-controller-publish: | |
| name: Publish RS controller to PyPI | |
| needs: rs-controller-wheels | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' | |
| steps: | |
| - name: Download all wheel artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: rs-controller-wheel-* | |
| merge-multiple: true | |
| path: dist/ | |
| - name: Install twine | |
| run: pip install twine | |
| - name: Publish to PyPI | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| twine upload --verbose --skip-existing dist/* | |
| - name: Wait for PyPI availability | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| LINK="https://pypi.org/project/flyte_controller_base/${VERSION}/" | |
| echo "Waiting for $LINK" | |
| for i in $(seq 1 60); do | |
| if curl -L -I -s -f "$LINK"; then | |
| echo "Found on PyPI: $LINK" | |
| exit 0 | |
| else | |
| echo "Attempt $i: not yet available, retrying in 10s..." | |
| sleep 10 | |
| fi | |
| done | |
| echo "ERROR: timed out waiting for PyPI" | |
| exit 1 | |
| flyte-pypi: | |
| name: PyPI package | |
| needs: rs-controller-publish | |
| if: always() && (needs.rs-controller-publish.result == 'success' || needs.rs-controller-publish.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.0 | |
| id: setup-uv | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| - name: Pin flyte_controller_base version (release only) | |
| if: github.event_name == 'release' | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| sed -i "s/flyte_controller_base>=2.0.0b0/flyte_controller_base==${VERSION}/" pyproject.toml | |
| echo "Pinned flyte_controller_base==${VERSION}" | |
| grep flyte_controller_base pyproject.toml | |
| # Editing pyproject.toml above makes the git working tree dirty, which | |
| # would cause setuptools_scm to produce a PEP 440 local version like | |
| # "2.3.1.dev0+g<sha>.d<date>" that PyPI rejects. Pin the version | |
| # explicitly to the release tag so the resulting wheel is uploadable. | |
| echo "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTE=${VERSION}" >> $GITHUB_ENV | |
| - name: Build and publish | |
| run: | | |
| uv run python -m build --wheel --installer uv | |
| - name: Publish | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| uvx twine upload --verbose --skip-existing dist/* | |
| - name: Sleep until pypi is available | |
| if: ${{ github.event_name == 'release' }} | |
| id: pypiwait | |
| run: | | |
| # from 'v1.2.3' get '1.2.3' and make sure it's not an empty string | |
| VERSION="${{ github.ref_name }}" | |
| VERSION="${VERSION#v}" | |
| if [ -z "$VERSION" ] | |
| then | |
| echo "No tagged version found, exiting" | |
| exit 1 | |
| fi | |
| sleep 300 | |
| LINK="https://pypi.org/project/flyte/${VERSION}/" | |
| for i in {1..60}; do | |
| result=$(curl -L -I -s -f ${LINK}) | |
| if [ $? -eq 0 ]; then | |
| echo "Found pypi for $LINK" | |
| exit 0 | |
| else | |
| echo "Did not find - Retrying in 10 seconds..." | |
| sleep 10 | |
| fi | |
| done | |
| exit 1 | |
| shell: bash | |
| flyte-constraints: | |
| # Publish the tested dependency resolution from uv.lock as an optional constraints file. | |
| name: Publish dependency constraints | |
| needs: flyte-pypi | |
| if: always() && needs.flyte-pypi.result == 'success' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.0 | |
| with: | |
| version: "0.9.21" | |
| - name: Export constraints from uv.lock | |
| run: | | |
| if [ "${{ github.event_name }}" = "release" ]; then | |
| VERSION="${{ github.ref_name }}"; VERSION="${VERSION#v}" | |
| else | |
| VERSION="0.0.0.dev0" # non-release runs (e.g. PR validation) | |
| fi | |
| # Universal lock: one exported file is valid across all Python versions and platforms. | |
| uv export \ | |
| --frozen \ | |
| --no-dev \ | |
| --no-emit-project \ | |
| --no-hashes \ | |
| --no-annotate \ | |
| --no-header \ | |
| -o constraints-body.txt | |
| { | |
| echo "# Tested dependency versions for flyte==${VERSION}." | |
| echo "# Optional: pip install flyte==${VERSION} -c constraints-${VERSION}.txt" | |
| } > "constraints-${VERSION}.txt" | |
| cat constraints-body.txt >> "constraints-${VERSION}.txt" | |
| # Unversioned copy so releases/latest/download/constraints.txt resolves. | |
| cp "constraints-${VERSION}.txt" constraints.txt | |
| echo "::group::constraints-${VERSION}.txt" | |
| cat "constraints-${VERSION}.txt" | |
| echo "::endgroup::" | |
| - name: Upload constraints as a workflow artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: flyte-constraints | |
| path: constraints*.txt | |
| if-no-files-found: error | |
| - name: Attach constraints to the GitHub Release | |
| if: github.event_name == 'release' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| VERSION="${{ github.ref_name }}" | |
| VERSION="${VERSION#v}" | |
| gh release upload "${{ github.ref_name }}" \ | |
| "constraints-${VERSION}.txt" constraints.txt --clobber | |
| plugin-pypi: | |
| name: PyPI package | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| workdir: | |
| - "plugins/ray" | |
| - "plugins/spark" | |
| - "plugins/dask" | |
| - "plugins/pytorch" | |
| - "plugins/bigquery" | |
| - "plugins/databricks" | |
| - "plugins/snowflake" | |
| - "plugins/sglang" | |
| - "plugins/vllm" | |
| - "plugins/wandb" | |
| - "plugins/polars" | |
| - "plugins/codegen" | |
| - "plugins/hitl" | |
| - "plugins/jsonl" | |
| - "plugins/mlflow" | |
| - "plugins/papermill" | |
| - "plugins/pandera" | |
| - "plugins/hydra" | |
| - "plugins/omegaconf" | |
| - "plugins/huggingface" | |
| - "plugins/otel" | |
| - "plugins/agents/core" | |
| - "plugins/agents/openai" | |
| - "plugins/agents/claude" | |
| - "plugins/agents/mistral" | |
| - "plugins/agents/google" | |
| - "plugins/agents/crewai" | |
| - "plugins/agents/langchain" | |
| - "plugins/agents/deepagents" | |
| - "plugins/agents/langgraph" | |
| - "plugins/agents/pydantic_ai" | |
| - "plugins/agents/hermes" | |
| - "plugins/lance" | |
| include: | |
| - workdir: "plugins/sglang" | |
| image-type: sglang | |
| - workdir: "plugins/vllm" | |
| image-type: vllm | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.0 | |
| id: setup-uv | |
| - name: Set version from tag (release only) | |
| if: github.event_name == 'release' | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| # `uv run` triggers `uv sync`, which may re-resolve and rewrite the | |
| # plugin's uv.lock (e.g. for plugins that pull `flyte` from PyPI rather | |
| # than via `[tool.uv.sources]`). That dirties the working tree, which | |
| # would cause setuptools_scm (root = "../../") to emit a PEP 440 local | |
| # version like "2.3.0b1.dev0+g<sha>.d<date>" that PyPI rejects. Pinning | |
| # the version explicitly bypasses git inspection entirely. | |
| echo "SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}" >> $GITHUB_ENV | |
| echo "Pinned SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}" | |
| - name: Install dependencies | |
| working-directory: ${{ matrix.workdir }} | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| - name: Build and publish | |
| working-directory: ${{ matrix.workdir }} | |
| run: | | |
| uv run python -m build --wheel --installer uv | |
| - name: Publish | |
| if: ${{ github.event_name == 'release' }} | |
| working-directory: ${{ matrix.workdir }} | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| uvx twine upload --verbose --skip-existing dist/* | |
| build-and-push-flyte-docker-images: | |
| needs: [flyte-pypi, rs-controller-wheels] | |
| if: always() && needs.flyte-pypi.result == 'success' && needs.rs-controller-wheels.result == 'success' | |
| name: Flyte image for Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| # The SDK never defaults the *push* registry to ghcr.io/flyteorg (end users cannot push | |
| # there), so this job — which does log in to it below — has to name it explicitly. | |
| env: | |
| FLYTE_IMAGE_REGISTRY: ghcr.io/flyteorg | |
| strategy: | |
| matrix: | |
| python-version: | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| - "3.14" | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v4 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ghcr.io/flyteorg | |
| username: "${{ secrets.FLYTE_BOT_USERNAME }}" | |
| password: "${{ secrets.FLYTE_BOT_PAT }}" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-python-env | |
| with: | |
| python-version: "${{ matrix.python-version }}" | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| uv pip freeze | |
| - name: Build wheel | |
| run: | | |
| make dist | |
| - name: Download rs-controller wheels | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: rs-controller-wheel-* | |
| merge-multiple: true | |
| path: rs_controller/dist/ | |
| - name: Build and push the flyte image | |
| env: | |
| FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha" | |
| FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max" | |
| _F_USE_RUST_CONTROLLER: "1" | |
| run: | | |
| uv run python maint_tools/build_default_image.py --type flyte | |
| - name: Build and push the connector image | |
| env: | |
| FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha" | |
| FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max" | |
| run: | | |
| uv run python maint_tools/build_default_image.py --type connector | |
| notify-docs: | |
| name: Signal unionai-docs to regenerate API docs | |
| needs: [flyte-pypi, plugin-pypi] | |
| # Fire only on a real release once the core `flyte` package is on PyPI. | |
| # Tolerates partial plugin-publish failures (docs regen picks up whatever is | |
| # live on PyPI). Mirrors the always()+result gating used by the image job. | |
| if: always() && github.event_name == 'release' && needs.flyte-pypi.result == 'success' | |
| runs-on: ubuntu-latest | |
| # Job-level env so the App id is referenceable in step `if:` (secrets aren't). | |
| env: | |
| DOCSY_BOT_APP_ID: ${{ secrets.DOCSY_BOT_APP_ID }} | |
| steps: | |
| # Authenticate as the "docsy-bot" GitHub App and mint a short-lived token | |
| # scoped to unionai/unionai-docs (where the App is installed). The App is | |
| # NOT installed on this repo — we only hold its credentials as secrets to | |
| # request a cross-org installation token. Skipped until the secrets exist. | |
| - name: Mint docsy-bot App token for unionai-docs | |
| id: app-token | |
| if: ${{ env.DOCSY_BOT_APP_ID != '' }} | |
| uses: actions/create-github-app-token@v3 | |
| with: | |
| app-id: ${{ secrets.DOCSY_BOT_APP_ID }} | |
| private-key: ${{ secrets.DOCSY_BOT_PRIVATE_KEY }} | |
| owner: unionai | |
| repositories: unionai-docs | |
| - name: Send repository_dispatch to unionai/unionai-docs | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| if [ -z "${GH_TOKEN}" ]; then | |
| echo "::warning title=docs regen signal skipped::docsy-bot App not configured (DOCSY_BOT_APP_ID / DOCSY_BOT_PRIVATE_KEY) — skipping the unionai-docs regen signal. Set the secrets to enable it." | |
| exit 0 | |
| fi | |
| VERSION="${GITHUB_REF#refs/tags/v}" | |
| echo "Signaling unionai/unionai-docs to regenerate API docs for flyte ${VERSION}" | |
| gh api repos/unionai/unionai-docs/dispatches \ | |
| -f event_type=sdk-release \ | |
| -f 'client_payload[version]='"${VERSION}" |