Claim new flyteplugins PyPI names at merge time, not release time #392
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish | |
| on: | |
| release: | |
| types: [published] | |
| workflow_dispatch: | |
| inputs: | |
| tag: | |
| description: "Release tag to rebuild, e.g. v2.6.0. Rebuilds and pushes the images for an already-published release; every PyPI upload step is release-gated, so nothing is re-published." | |
| required: true | |
| pull_request: | |
| paths: | |
| - ".github/workflows/publish.yml" | |
| - "maint_tools/build_default_image.py" | |
| env: | |
| # Release runs use the tag they fired on; manual reruns use the tag input. | |
| RELEASE_TAG: ${{ inputs.tag || github.ref_name }} | |
| jobs: | |
| rs-controller-wheels: | |
| name: Build RS controller wheel (${{ matrix.target }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| # one flaky target shouldn't cancel the wheels that already built | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - target: x86_64 | |
| os: ubuntu-latest | |
| - target: aarch64 | |
| os: ubuntu-24.04-arm | |
| - target: aarch64-apple-darwin | |
| os: macos-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Set version from tag | |
| run: | | |
| if [[ "$RELEASE_TAG" == v* ]]; then | |
| VERSION="${RELEASE_TAG#v}" | |
| else | |
| VERSION="0.0.0.dev0" | |
| fi | |
| echo "VERSION=$VERSION" >> $GITHUB_ENV | |
| sed "s/^version = .*/version = \"$VERSION\"/" rs_controller/pyproject.toml > tmp && mv tmp rs_controller/pyproject.toml | |
| echo "Set version to $VERSION" | |
| cat rs_controller/pyproject.toml | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: auto | |
| args: --release --out dist -m rs_controller/Cargo.toml | |
| sccache: true | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: rs-controller-wheel-${{ matrix.target }} | |
| path: dist/*.whl | |
| rs-controller-publish: | |
| name: Publish RS controller to PyPI | |
| needs: rs-controller-wheels | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' | |
| steps: | |
| - name: Download all wheel artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: rs-controller-wheel-* | |
| merge-multiple: true | |
| path: dist/ | |
| - name: Install twine | |
| run: pip install twine | |
| - name: Publish to PyPI | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| twine upload --verbose --skip-existing dist/* | |
| - name: Wait for PyPI availability | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| LINK="https://pypi.org/project/flyte_controller_base/${VERSION}/" | |
| echo "Waiting for $LINK" | |
| for i in $(seq 1 60); do | |
| if curl -L -I -s -f "$LINK"; then | |
| echo "Found on PyPI: $LINK" | |
| exit 0 | |
| else | |
| echo "Attempt $i: not yet available, retrying in 10s..." | |
| sleep 10 | |
| fi | |
| done | |
| echo "ERROR: timed out waiting for PyPI" | |
| exit 1 | |
| flyte-pypi: | |
| name: PyPI package | |
| needs: rs-controller-publish | |
| if: always() && (needs.rs-controller-publish.result == 'success' || needs.rs-controller-publish.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.0 | |
| id: setup-uv | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| - name: Pin flyte_controller_base version (release only) | |
| if: github.event_name == 'release' | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| sed -i "s/flyte_controller_base>=2.0.0b0/flyte_controller_base==${VERSION}/" pyproject.toml | |
| echo "Pinned flyte_controller_base==${VERSION}" | |
| grep flyte_controller_base pyproject.toml | |
| # Editing pyproject.toml above makes the git working tree dirty, which | |
| # would cause setuptools_scm to produce a PEP 440 local version like | |
| # "2.3.1.dev0+g<sha>.d<date>" that PyPI rejects. Pin the version | |
| # explicitly to the release tag so the resulting wheel is uploadable. | |
| echo "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTE=${VERSION}" >> $GITHUB_ENV | |
| - name: Build and publish | |
| run: | | |
| uv run python -m build --wheel --installer uv | |
| - name: Publish | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| uvx twine upload --verbose --skip-existing dist/* | |
| - name: Sleep until pypi is available | |
| if: ${{ github.event_name == 'release' }} | |
| id: pypiwait | |
| run: | | |
| # from 'v1.2.3' get '1.2.3' and make sure it's not an empty string | |
| VERSION="${{ github.ref_name }}" | |
| VERSION="${VERSION#v}" | |
| if [ -z "$VERSION" ] | |
| then | |
| echo "No tagged version found, exiting" | |
| exit 1 | |
| fi | |
| sleep 300 | |
| LINK="https://pypi.org/project/flyte/${VERSION}/" | |
| for i in {1..60}; do | |
| result=$(curl -L -I -s -f ${LINK}) | |
| if [ $? -eq 0 ]; then | |
| echo "Found pypi for $LINK" | |
| exit 0 | |
| else | |
| echo "Did not find - Retrying in 10 seconds..." | |
| sleep 10 | |
| fi | |
| done | |
| exit 1 | |
| shell: bash | |
| flyte-constraints: | |
| # Publish the tested dependency resolution from uv.lock as an optional constraints file. | |
| name: Publish dependency constraints | |
| needs: flyte-pypi | |
| if: always() && needs.flyte-pypi.result == 'success' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.0 | |
| with: | |
| version: "0.9.21" | |
| - name: Export constraints from uv.lock | |
| run: | | |
| if [ "${{ github.event_name }}" = "release" ]; then | |
| VERSION="${{ github.ref_name }}"; VERSION="${VERSION#v}" | |
| else | |
| VERSION="0.0.0.dev0" # non-release runs (e.g. PR validation) | |
| fi | |
| # Universal lock: one exported file is valid across all Python versions and platforms. | |
| uv export \ | |
| --frozen \ | |
| --no-dev \ | |
| --no-emit-project \ | |
| --no-hashes \ | |
| --no-annotate \ | |
| --no-header \ | |
| -o constraints-body.txt | |
| { | |
| echo "# Tested dependency versions for flyte==${VERSION}." | |
| echo "# Optional: pip install flyte==${VERSION} -c constraints-${VERSION}.txt" | |
| } > "constraints-${VERSION}.txt" | |
| cat constraints-body.txt >> "constraints-${VERSION}.txt" | |
| # Unversioned copy so releases/latest/download/constraints.txt resolves. | |
| cp "constraints-${VERSION}.txt" constraints.txt | |
| echo "::group::constraints-${VERSION}.txt" | |
| cat "constraints-${VERSION}.txt" | |
| echo "::endgroup::" | |
| - name: Upload constraints as a workflow artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: flyte-constraints | |
| path: constraints*.txt | |
| if-no-files-found: error | |
| - name: Attach constraints to the GitHub Release | |
| if: github.event_name == 'release' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| VERSION="${{ github.ref_name }}" | |
| VERSION="${VERSION#v}" | |
| gh release upload "${{ github.ref_name }}" \ | |
| "constraints-${VERSION}.txt" constraints.txt --clobber | |
| discover-plugin-packages: | |
| name: Discover plugin packages | |
| runs-on: ubuntu-latest | |
| outputs: | |
| workdirs: ${{ steps.find.outputs.workdirs }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Find every distribution that claims a flyteplugins name | |
| id: find | |
| run: | | |
| # Every directory whose pyproject.toml claims a flyteplugins name has reserved that | |
| # name on PyPI, so it has to be published from here. PyPI is first come first served | |
| # and there is no ownership of the flyteplugins prefix, so a name we declare but never | |
| # upload just sits unregistered for anyone to claim -- and because pip runs a source | |
| # distribution's build backend at install time, whoever claims it executes code | |
| # wherever our docs tell people to install it, including our own remote image builder. | |
| # This job used to be a hand-maintained list, which silently drifted from the plugins | |
| # actually in the tree. Discovering the set instead means adding a plugin cannot leave | |
| # its name unclaimed. maxdepth 3 covers plugins/<name> and plugins/agents/<name>. | |
| # claim-plugin-names.yml registers each new name on PyPI the day its PR merges (PyPI | |
| # caps new-project creation at 4 per account per 24h), so by release time every name | |
| # here should already exist and this workflow only uploads to existing projects. | |
| workdirs=$(find plugins -mindepth 1 -maxdepth 3 -name pyproject.toml -not -path '*/.venv/*' \ | |
| -exec grep -lE '^name[[:space:]]*=[[:space:]]*"flyteplugins' {} + \ | |
| | xargs -n1 dirname | sort | jq -R -s -c 'split("\n") | map(select(. != ""))') | |
| if [ "$(echo "$workdirs" | jq 'length')" -eq 0 ]; then | |
| echo "ERROR: discovered no plugin distributions; refusing to publish an empty set" | |
| exit 1 | |
| fi | |
| echo "Discovered $(echo "$workdirs" | jq 'length') plugin distributions:" | |
| echo "$workdirs" | jq -r '.[]' | |
| echo "workdirs=$workdirs" >> "$GITHUB_OUTPUT" | |
| plugin-pypi: | |
| name: PyPI package | |
| needs: discover-plugin-packages | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| workdir: ${{ fromJson(needs.discover-plugin-packages.outputs.workdirs) }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v10.0.0 | |
| id: setup-uv | |
| - name: Set version from tag (release only) | |
| if: github.event_name == 'release' | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| # `uv run` triggers `uv sync`, which may re-resolve and rewrite the | |
| # plugin's uv.lock (e.g. for plugins that pull `flyte` from PyPI rather | |
| # than via `[tool.uv.sources]`). That dirties the working tree, which | |
| # would cause setuptools_scm (root = "../../") to emit a PEP 440 local | |
| # version like "2.3.0b1.dev0+g<sha>.d<date>" that PyPI rejects. Pinning | |
| # the version explicitly bypasses git inspection entirely. | |
| echo "SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}" >> $GITHUB_ENV | |
| echo "Pinned SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}" | |
| - name: Install dependencies | |
| working-directory: ${{ matrix.workdir }} | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| - name: Build and publish | |
| working-directory: ${{ matrix.workdir }} | |
| run: | | |
| uv run python -m build --wheel --installer uv | |
| - name: Publish | |
| if: ${{ github.event_name == 'release' }} | |
| working-directory: ${{ matrix.workdir }} | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| uvx twine upload --verbose --skip-existing dist/* | |
| build-and-push-flyte-docker-images: | |
| needs: [flyte-pypi, rs-controller-wheels] | |
| if: always() && needs.flyte-pypi.result == 'success' && needs.rs-controller-wheels.result == 'success' | |
| name: Flyte image for Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| # The SDK never defaults the *push* registry to ghcr.io/flyteorg (end users cannot push | |
| # there), so this job — which does log in to it below — has to name it explicitly. | |
| env: | |
| FLYTE_IMAGE_REGISTRY: ghcr.io/flyteorg | |
| strategy: | |
| matrix: | |
| python-version: | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| - "3.14" | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: "0" | |
| ref: ${{ inputs.tag || github.ref }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v4 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ghcr.io/flyteorg | |
| username: "${{ secrets.FLYTE_BOT_USERNAME }}" | |
| password: "${{ secrets.FLYTE_BOT_PAT }}" | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-python-env | |
| with: | |
| python-version: "${{ matrix.python-version }}" | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| uv pip freeze | |
| - name: Build wheel | |
| run: | | |
| make dist | |
| - name: Download rs-controller wheels | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: rs-controller-wheel-* | |
| merge-multiple: true | |
| path: rs_controller/dist/ | |
| - name: Build and push the flyte image | |
| env: | |
| FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha" | |
| FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max" | |
| _F_USE_RUST_CONTROLLER: "1" | |
| run: | | |
| uv run python maint_tools/build_default_image.py --type flyte | |
| - name: Build and push the connector image | |
| env: | |
| FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha" | |
| FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max" | |
| run: | | |
| uv run python maint_tools/build_default_image.py --type connector | |
| notify-docs: | |
| name: Signal unionai-docs to regenerate API docs | |
| needs: [flyte-pypi, plugin-pypi] | |
| # Fire only on a real release once the core `flyte` package is on PyPI. | |
| # Tolerates partial plugin-publish failures (docs regen picks up whatever is | |
| # live on PyPI). Mirrors the always()+result gating used by the image job. | |
| if: always() && github.event_name == 'release' && needs.flyte-pypi.result == 'success' | |
| runs-on: ubuntu-latest | |
| # Job-level env so the App id is referenceable in step `if:` (secrets aren't). | |
| env: | |
| DOCSY_BOT_APP_ID: ${{ secrets.DOCSY_BOT_APP_ID }} | |
| steps: | |
| # Authenticate as the "docsy-bot" GitHub App and mint a short-lived token | |
| # scoped to unionai/unionai-docs (where the App is installed). The App is | |
| # NOT installed on this repo — we only hold its credentials as secrets to | |
| # request a cross-org installation token. Skipped until the secrets exist. | |
| - name: Mint docsy-bot App token for unionai-docs | |
| id: app-token | |
| if: ${{ env.DOCSY_BOT_APP_ID != '' }} | |
| uses: actions/create-github-app-token@v3 | |
| with: | |
| app-id: ${{ secrets.DOCSY_BOT_APP_ID }} | |
| private-key: ${{ secrets.DOCSY_BOT_PRIVATE_KEY }} | |
| owner: unionai | |
| repositories: unionai-docs | |
| - name: Send repository_dispatch to unionai/unionai-docs | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| if [ -z "${GH_TOKEN}" ]; then | |
| echo "::warning title=docs regen signal skipped::docsy-bot App not configured (DOCSY_BOT_APP_ID / DOCSY_BOT_PRIVATE_KEY) — skipping the unionai-docs regen signal. Set the secrets to enable it." | |
| exit 0 | |
| fi | |
| VERSION="${GITHUB_REF#refs/tags/v}" | |
| echo "Signaling unionai/unionai-docs to regenerate API docs for flyte ${VERSION}" | |
| gh api repos/unionai/unionai-docs/dispatches \ | |
| -f event_type=sdk-release \ | |
| -f 'client_payload[version]='"${VERSION}" |