publish.yml

name: Publish

on:
  release:
    types: [published]
  workflow_dispatch:
    inputs:
      tag:
        description: "Release tag to rebuild, e.g. v2.6.0. Rebuilds and pushes the images for an already-published release; every PyPI upload step is release-gated, so nothing is re-published."
        required: true
  pull_request:
    paths:
      - ".github/workflows/publish.yml"
      - "maint_tools/build_default_image.py"

env:
  # Release runs use the tag they fired on; manual reruns use the tag input.
  RELEASE_TAG: ${{ inputs.tag || github.ref_name }}

jobs:
  rs-controller-wheels:
    name: Build RS controller wheel (${{ matrix.target }})
    runs-on: ${{ matrix.os }}
    strategy:
      # one flaky target shouldn't cancel the wheels that already built
      fail-fast: false
      matrix:
        include:
          - target: x86_64
            os: ubuntu-latest
          - target: aarch64
            os: ubuntu-24.04-arm
          - target: aarch64-apple-darwin
            os: macos-latest
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: "0"
          ref: ${{ inputs.tag || github.ref }}
      - name: Set version from tag
        run: |
          if [[ "$RELEASE_TAG" == v* ]]; then
            VERSION="${RELEASE_TAG#v}"
          else
            VERSION="0.0.0.dev0"
          fi
          echo "VERSION=$VERSION" >> $GITHUB_ENV
          sed "s/^version = .*/version = \"$VERSION\"/" rs_controller/pyproject.toml > tmp && mv tmp rs_controller/pyproject.toml
          echo "Set version to $VERSION"
          cat rs_controller/pyproject.toml
      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: auto
          args: --release --out dist -m rs_controller/Cargo.toml
          sccache: true
      - name: Upload wheels
        uses: actions/upload-artifact@v7
        with:
          name: rs-controller-wheel-${{ matrix.target }}
          path: dist/*.whl

  rs-controller-publish:
    name: Publish RS controller to PyPI
    needs: rs-controller-wheels
    runs-on: ubuntu-latest
    if: github.event_name == 'release'
    steps:
      - name: Download all wheel artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: rs-controller-wheel-*
          merge-multiple: true
          path: dist/
      - name: Install twine
        run: pip install twine
      - name: Publish to PyPI
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          twine upload --verbose --skip-existing dist/*
      - name: Wait for PyPI availability
        run: |
          VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||')
          LINK="https://pypi.org/project/flyte_controller_base/${VERSION}/"
          echo "Waiting for $LINK"
          for i in $(seq 1 60); do
            if curl -L -I -s -f "$LINK"; then
              echo "Found on PyPI: $LINK"
              exit 0
            else
              echo "Attempt $i: not yet available, retrying in 10s..."
              sleep 10
            fi
          done
          echo "ERROR: timed out waiting for PyPI"
          exit 1

  flyte-pypi:
    name: PyPI package
    needs: rs-controller-publish
    if: always() && (needs.rs-controller-publish.result == 'success' || needs.rs-controller-publish.result == 'skipped')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: "0"
          ref: ${{ inputs.tag || github.ref }}
      - name: Set up Python
        uses: actions/setup-python@v7
        with:
          python-version: "3.13"
      - name: Install uv
        uses: astral-sh/setup-uv@v10.0.0
        id: setup-uv
      - name: Install dependencies
        run: |
          uv venv
          uv pip install build twine setuptools wheel
      - name: Pin flyte_controller_base version (release only)
        if: github.event_name == 'release'
        run: |
          VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||')
          sed -i "s/flyte_controller_base>=2.0.0b0/flyte_controller_base==${VERSION}/" pyproject.toml
          echo "Pinned flyte_controller_base==${VERSION}"
          grep flyte_controller_base pyproject.toml
          # Editing pyproject.toml above makes the git working tree dirty, which
          # would cause setuptools_scm to produce a PEP 440 local version like
          # "2.3.1.dev0+g<sha>.d<date>" that PyPI rejects. Pin the version
          # explicitly to the release tag so the resulting wheel is uploadable.
          echo "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTE=${VERSION}" >> $GITHUB_ENV
      - name: Build and publish
        run: |
          uv run python -m build --wheel --installer uv
      - name: Publish
        if: ${{ github.event_name == 'release' }}
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          uvx twine upload --verbose --skip-existing dist/*
      - name: Sleep until pypi is available
        if: ${{ github.event_name == 'release' }}
        id: pypiwait
        run: |
          # from 'v1.2.3' get '1.2.3' and make sure it's not an empty string
          VERSION="${{ github.ref_name }}"
          VERSION="${VERSION#v}"
          if [ -z "$VERSION" ]
          then
            echo "No tagged version found, exiting"
            exit 1
          fi
          sleep 300
          LINK="https://pypi.org/project/flyte/${VERSION}/"
          for i in {1..60}; do
            result=$(curl -L -I -s -f ${LINK})
            if [ $? -eq 0 ]; then
              echo "Found pypi for $LINK"
              exit 0
            else
              echo "Did not find - Retrying in 10 seconds..."
              sleep 10
            fi
          done
          exit 1
        shell: bash

  flyte-constraints:
    # Publish the tested dependency resolution from uv.lock as an optional constraints file.
    name: Publish dependency constraints
    needs: flyte-pypi
    if: always() && needs.flyte-pypi.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: "0"
          ref: ${{ inputs.tag || github.ref }}
      - name: Install uv
        uses: astral-sh/setup-uv@v10.0.0
        with:
          version: "0.9.21"
      - name: Export constraints from uv.lock
        run: |
          if [ "${{ github.event_name }}" = "release" ]; then
            VERSION="${{ github.ref_name }}"; VERSION="${VERSION#v}"
          else
            VERSION="0.0.0.dev0"  # non-release runs (e.g. PR validation)
          fi
          # Universal lock: one exported file is valid across all Python versions and platforms.
          uv export \
            --frozen \
            --no-dev \
            --no-emit-project \
            --no-hashes \
            --no-annotate \
            --no-header \
            -o constraints-body.txt
          {
            echo "# Tested dependency versions for flyte==${VERSION}."
            echo "# Optional: pip install flyte==${VERSION} -c constraints-${VERSION}.txt"
          } > "constraints-${VERSION}.txt"
          cat constraints-body.txt >> "constraints-${VERSION}.txt"
          # Unversioned copy so releases/latest/download/constraints.txt resolves.
          cp "constraints-${VERSION}.txt" constraints.txt
          echo "::group::constraints-${VERSION}.txt"
          cat "constraints-${VERSION}.txt"
          echo "::endgroup::"
      - name: Upload constraints as a workflow artifact
        uses: actions/upload-artifact@v7
        with:
          name: flyte-constraints
          path: constraints*.txt
          if-no-files-found: error
      - name: Attach constraints to the GitHub Release
        if: github.event_name == 'release'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          VERSION="${{ github.ref_name }}"
          VERSION="${VERSION#v}"
          gh release upload "${{ github.ref_name }}" \
            "constraints-${VERSION}.txt" constraints.txt --clobber

  discover-plugin-packages:
    name: Discover plugin packages
    runs-on: ubuntu-latest
    outputs:
      workdirs: ${{ steps.find.outputs.workdirs }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: "0"
          ref: ${{ inputs.tag || github.ref }}
      - name: Find every distribution that claims a flyteplugins name
        id: find
        run: |
          # Every directory whose pyproject.toml claims a flyteplugins name has reserved that
          # name on PyPI, so it has to be published from here. PyPI is first come first served
          # and there is no ownership of the flyteplugins prefix, so a name we declare but never
          # upload just sits unregistered for anyone to claim -- and because pip runs a source
          # distribution's build backend at install time, whoever claims it executes code
          # wherever our docs tell people to install it, including our own remote image builder.
          # This job used to be a hand-maintained list, which silently drifted from the plugins
          # actually in the tree. Discovering the set instead means adding a plugin cannot leave
          # its name unclaimed. maxdepth 3 covers plugins/<name> and plugins/agents/<name>.
          # claim-plugin-names.yml registers each new name on PyPI the day its PR merges (PyPI
          # caps new-project creation at 4 per account per 24h), so by release time every name
          # here should already exist and this workflow only uploads to existing projects.
          workdirs=$(find plugins -mindepth 1 -maxdepth 3 -name pyproject.toml -not -path '*/.venv/*' \
            -exec grep -lE '^name[[:space:]]*=[[:space:]]*"flyteplugins' {} + \
            | xargs -n1 dirname | sort | jq -R -s -c 'split("\n") | map(select(. != ""))')
          if [ "$(echo "$workdirs" | jq 'length')" -eq 0 ]; then
            echo "ERROR: discovered no plugin distributions; refusing to publish an empty set"
            exit 1
          fi
          echo "Discovered $(echo "$workdirs" | jq 'length') plugin distributions:"
          echo "$workdirs" | jq -r '.[]'
          echo "workdirs=$workdirs" >> "$GITHUB_OUTPUT"

  plugin-pypi:
    name: PyPI package
    needs: discover-plugin-packages
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        workdir: ${{ fromJson(needs.discover-plugin-packages.outputs.workdirs) }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: "0"
          ref: ${{ inputs.tag || github.ref }}
      - name: Set up Python
        uses: actions/setup-python@v7
        with:
          python-version: "3.13"
      - name: Install uv
        uses: astral-sh/setup-uv@v10.0.0
        id: setup-uv
      - name: Set version from tag (release only)
        if: github.event_name == 'release'
        run: |
          VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||')
          # `uv run` triggers `uv sync`, which may re-resolve and rewrite the
          # plugin's uv.lock (e.g. for plugins that pull `flyte` from PyPI rather
          # than via `[tool.uv.sources]`). That dirties the working tree, which
          # would cause setuptools_scm (root = "../../") to emit a PEP 440 local
          # version like "2.3.0b1.dev0+g<sha>.d<date>" that PyPI rejects. Pinning
          # the version explicitly bypasses git inspection entirely.
          echo "SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}" >> $GITHUB_ENV
          echo "Pinned SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}"
      - name: Install dependencies
        working-directory: ${{ matrix.workdir }}
        run: |
          uv venv
          uv pip install build twine setuptools wheel
      - name: Build and publish
        working-directory: ${{ matrix.workdir }}
        run: |
          uv run python -m build --wheel --installer uv
      - name: Publish
        if: ${{ github.event_name == 'release' }}
        working-directory: ${{ matrix.workdir }}
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          uvx twine upload --verbose --skip-existing dist/*

  build-and-push-flyte-docker-images:
    needs: [flyte-pypi, rs-controller-wheels]
    if: always() && needs.flyte-pypi.result == 'success' && needs.rs-controller-wheels.result == 'success'
    name: Flyte image for Python ${{ matrix.python-version }}
    runs-on: ubuntu-latest
    # The SDK never defaults the *push* registry to ghcr.io/flyteorg (end users cannot push
    # there), so this job — which does log in to it below — has to name it explicitly.
    env:
      FLYTE_IMAGE_REGISTRY: ghcr.io/flyteorg
    strategy:
      matrix:
        python-version:
          - "3.10"
          - "3.11"
          - "3.12"
          - "3.13"
          - "3.14"
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: "0"
          ref: ${{ inputs.tag || github.ref }}
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v4
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v4
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io/flyteorg
          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
          password: "${{ secrets.FLYTE_BOT_PAT }}"
      - name: Set up Python ${{ matrix.python-version }}
        uses: ./.github/actions/setup-python-env
        with:
          python-version: "${{ matrix.python-version }}"
      - name: Install dependencies
        run: |
          uv venv
          uv pip install build twine setuptools wheel
          uv pip freeze
      - name: Build wheel
        run: |
          make dist
      - name: Download rs-controller wheels
        uses: actions/download-artifact@v8
        with:
          pattern: rs-controller-wheel-*
          merge-multiple: true
          path: rs_controller/dist/
      - name: Build and push the flyte image
        env:
          FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha"
          FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max"
          _F_USE_RUST_CONTROLLER: "1"
        run: |
          uv run python maint_tools/build_default_image.py --type flyte
      - name: Build and push the connector image
        env:
          FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha"
          FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max"
        run: |
          uv run python maint_tools/build_default_image.py --type connector

  notify-docs:
    name: Signal unionai-docs to regenerate API docs
    needs: [flyte-pypi, plugin-pypi]
    # Fire only on a real release once the core `flyte` package is on PyPI.
    # Tolerates partial plugin-publish failures (docs regen picks up whatever is
    # live on PyPI). Mirrors the always()+result gating used by the image job.
    if: always() && github.event_name == 'release' && needs.flyte-pypi.result == 'success'
    runs-on: ubuntu-latest
    # Job-level env so the App id is referenceable in step `if:` (secrets aren't).
    env:
      DOCSY_BOT_APP_ID: ${{ secrets.DOCSY_BOT_APP_ID }}
    steps:
      # Authenticate as the "docsy-bot" GitHub App and mint a short-lived token
      # scoped to unionai/unionai-docs (where the App is installed). The App is
      # NOT installed on this repo — we only hold its credentials as secrets to
      # request a cross-org installation token. Skipped until the secrets exist.
      - name: Mint docsy-bot App token for unionai-docs
        id: app-token
        if: ${{ env.DOCSY_BOT_APP_ID != '' }}
        uses: actions/create-github-app-token@v3
        with:
          app-id: ${{ secrets.DOCSY_BOT_APP_ID }}
          private-key: ${{ secrets.DOCSY_BOT_PRIVATE_KEY }}
          owner: unionai
          repositories: unionai-docs

      - name: Send repository_dispatch to unionai/unionai-docs
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
        run: |
          if [ -z "${GH_TOKEN}" ]; then
            echo "::warning title=docs regen signal skipped::docsy-bot App not configured (DOCSY_BOT_APP_ID / DOCSY_BOT_PRIVATE_KEY) — skipping the unionai-docs regen signal. Set the secrets to enable it."
            exit 0
          fi
          VERSION="${GITHUB_REF#refs/tags/v}"
          echo "Signaling unionai/unionai-docs to regenerate API docs for flyte ${VERSION}"
          gh api repos/unionai/unionai-docs/dispatches \
            -f event_type=sdk-release \
            -f 'client_payload[version]='"${VERSION}"