add lance plugin #345
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish | |
| on: | |
| release: | |
| types: [published] | |
| pull_request: | |
| paths: | |
| - ".github/workflows/publish.yml" | |
| - "maint_tools/build_default_image.py" | |
| jobs: | |
| rs-controller-wheels: | |
| name: Build RS controller wheel (${{ matrix.target }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - target: x86_64 | |
| os: ubuntu-latest | |
| - target: aarch64 | |
| os: ubuntu-24.04-arm | |
| - target: aarch64-apple-darwin | |
| os: macos-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set version from tag | |
| run: | | |
| if [[ "$GITHUB_REF" == refs/tags/v* ]]; then | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| else | |
| VERSION="0.0.0.dev0" | |
| fi | |
| echo "VERSION=$VERSION" >> $GITHUB_ENV | |
| sed "s/^version = .*/version = \"$VERSION\"/" rs_controller/pyproject.toml > tmp && mv tmp rs_controller/pyproject.toml | |
| echo "Set version to $VERSION" | |
| cat rs_controller/pyproject.toml | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: auto | |
| args: --release --out dist -m rs_controller/Cargo.toml | |
| sccache: true | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: rs-controller-wheel-${{ matrix.target }} | |
| path: dist/*.whl | |
| rs-controller-publish: | |
| name: Publish RS controller to PyPI | |
| needs: rs-controller-wheels | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' | |
| steps: | |
| - name: Download all wheel artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: rs-controller-wheel-* | |
| merge-multiple: true | |
| path: dist/ | |
| - name: Install twine | |
| run: pip install twine | |
| - name: Publish to PyPI | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| twine upload --verbose dist/* | |
| - name: Wait for PyPI availability | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| LINK="https://pypi.org/project/flyte_controller_base/${VERSION}/" | |
| echo "Waiting for $LINK" | |
| for i in $(seq 1 60); do | |
| if curl -L -I -s -f "$LINK"; then | |
| echo "Found on PyPI: $LINK" | |
| exit 0 | |
| else | |
| echo "Attempt $i: not yet available, retrying in 10s..." | |
| sleep 10 | |
| fi | |
| done | |
| echo "ERROR: timed out waiting for PyPI" | |
| exit 1 | |
| flyte-pypi: | |
| name: PyPI package | |
| needs: rs-controller-publish | |
| if: always() && (needs.rs-controller-publish.result == 'success' || needs.rs-controller-publish.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set up Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| id: setup-uv | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| - name: Pin flyte_controller_base version (release only) | |
| if: github.event_name == 'release' | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| sed -i "s/flyte_controller_base>=2.0.0b0/flyte_controller_base==${VERSION}/" pyproject.toml | |
| echo "Pinned flyte_controller_base==${VERSION}" | |
| grep flyte_controller_base pyproject.toml | |
| # Editing pyproject.toml above makes the git working tree dirty, which | |
| # would cause setuptools_scm to produce a PEP 440 local version like | |
| # "2.3.1.dev0+g<sha>.d<date>" that PyPI rejects. Pin the version | |
| # explicitly to the release tag so the resulting wheel is uploadable. | |
| echo "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTE=${VERSION}" >> $GITHUB_ENV | |
| - name: Build and publish | |
| run: | | |
| uv run python -m build --wheel --installer uv | |
| - name: Publish | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| uv run python -m twine upload --verbose dist/* | |
| - name: Sleep until pypi is available | |
| if: ${{ github.event_name == 'release' }} | |
| id: pypiwait | |
| run: | | |
| # from 'v1.2.3' get '1.2.3' and make sure it's not an empty string | |
| VERSION="${{ github.ref_name }}" | |
| VERSION="${VERSION#v}" | |
| if [ -z "$VERSION" ] | |
| then | |
| echo "No tagged version found, exiting" | |
| exit 1 | |
| fi | |
| sleep 300 | |
| LINK="https://pypi.org/project/flyte/${VERSION}/" | |
| for i in {1..60}; do | |
| result=$(curl -L -I -s -f ${LINK}) | |
| if [ $? -eq 0 ]; then | |
| echo "Found pypi for $LINK" | |
| exit 0 | |
| else | |
| echo "Did not find - Retrying in 10 seconds..." | |
| sleep 10 | |
| fi | |
| done | |
| exit 1 | |
| shell: bash | |
| plugin-pypi: | |
| name: PyPI package | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| workdir: | |
| - "plugins/ray" | |
| - "plugins/spark" | |
| - "plugins/dask" | |
| - "plugins/pytorch" | |
| - "plugins/bigquery" | |
| - "plugins/databricks" | |
| - "plugins/snowflake" | |
| - "plugins/sglang" | |
| - "plugins/vllm" | |
| - "plugins/wandb" | |
| - "plugins/polars" | |
| - "plugins/codegen" | |
| - "plugins/hitl" | |
| - "plugins/jsonl" | |
| - "plugins/mlflow" | |
| - "plugins/papermill" | |
| - "plugins/pandera" | |
| - "plugins/hydra" | |
| - "plugins/omegaconf" | |
| - "plugins/huggingface" | |
| - "plugins/agents/core" | |
| - "plugins/agents/openai" | |
| - "plugins/agents/claude" | |
| - "plugins/agents/mistral" | |
| - "plugins/agents/google" | |
| - "plugins/agents/crewai" | |
| - "plugins/agents/langchain" | |
| - "plugins/agents/deepagents" | |
| - "plugins/agents/langgraph" | |
| - "plugins/agents/pydantic_ai" | |
| - "plugins/agents/hermes" | |
| - "plugins/lance" | |
| include: | |
| - workdir: "plugins/sglang" | |
| image-type: sglang | |
| - workdir: "plugins/vllm" | |
| image-type: vllm | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set up Python | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| id: setup-uv | |
| - name: Set version from tag (release only) | |
| if: github.event_name == 'release' | |
| run: | | |
| VERSION=$(echo "$GITHUB_REF" | sed 's|refs/tags/v||') | |
| # `uv run` triggers `uv sync`, which may re-resolve and rewrite the | |
| # plugin's uv.lock (e.g. for plugins that pull `flyte` from PyPI rather | |
| # than via `[tool.uv.sources]`). That dirties the working tree, which | |
| # would cause setuptools_scm (root = "../../") to emit a PEP 440 local | |
| # version like "2.3.0b1.dev0+g<sha>.d<date>" that PyPI rejects. Pinning | |
| # the version explicitly bypasses git inspection entirely. | |
| echo "SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}" >> $GITHUB_ENV | |
| echo "Pinned SETUPTOOLS_SCM_PRETEND_VERSION=${VERSION}" | |
| - name: Install dependencies | |
| working-directory: ${{ matrix.workdir }} | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| - name: Build and publish | |
| working-directory: ${{ matrix.workdir }} | |
| run: | | |
| uv run python -m build --wheel --installer uv | |
| - name: Publish | |
| if: ${{ github.event_name == 'release' }} | |
| working-directory: ${{ matrix.workdir }} | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| uv run python -m twine upload --verbose dist/* | |
| build-and-push-flyte-docker-images: | |
| needs: [flyte-pypi, rs-controller-wheels] | |
| if: always() && needs.flyte-pypi.result == 'success' && needs.rs-controller-wheels.result == 'success' | |
| name: Flyte image for Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| - "3.14" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: "0" | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io/flyteorg | |
| username: "${{ secrets.FLYTE_BOT_USERNAME }}" | |
| password: "${{ secrets.FLYTE_BOT_PAT }}" | |
| - name: Fetch the code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: ./.github/actions/setup-python-env | |
| with: | |
| python-version: "${{ matrix.python-version }}" | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| uv pip install build twine setuptools wheel | |
| uv pip freeze | |
| - name: Build wheel | |
| run: | | |
| make dist | |
| - name: Download rs-controller wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: rs-controller-wheel-* | |
| merge-multiple: true | |
| path: rs_controller/dist/ | |
| - name: Build and push the flyte image | |
| env: | |
| FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha" | |
| FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max" | |
| _F_USE_RUST_CONTROLLER: "1" | |
| run: | | |
| uv run python maint_tools/build_default_image.py --type flyte | |
| - name: Build and push the connector image | |
| env: | |
| FLYTE_DOCKER_BUILDER_CACHE_FROM: "type=gha" | |
| FLYTE_DOCKER_BUILDER_CACHE_TO: "type=gha,mode=max" | |
| run: | | |
| uv run python maint_tools/build_default_image.py --type connector | |
| notify-docs: | |
| name: Signal unionai-docs to regenerate API docs | |
| needs: [flyte-pypi, plugin-pypi] | |
| # Fire only on a real release once the core `flyte` package is on PyPI. | |
| # Tolerates partial plugin-publish failures (docs regen picks up whatever is | |
| # live on PyPI). Mirrors the always()+result gating used by the image job. | |
| if: always() && github.event_name == 'release' && needs.flyte-pypi.result == 'success' | |
| runs-on: ubuntu-latest | |
| # Job-level env so the App id is referenceable in step `if:` (secrets aren't). | |
| env: | |
| DOCSY_BOT_APP_ID: ${{ secrets.DOCSY_BOT_APP_ID }} | |
| steps: | |
| # Authenticate as the "docsy-bot" GitHub App and mint a short-lived token | |
| # scoped to unionai/unionai-docs (where the App is installed). The App is | |
| # NOT installed on this repo — we only hold its credentials as secrets to | |
| # request a cross-org installation token. Skipped until the secrets exist. | |
| - name: Mint docsy-bot App token for unionai-docs | |
| id: app-token | |
| if: ${{ env.DOCSY_BOT_APP_ID != '' }} | |
| uses: actions/create-github-app-token@v1 | |
| with: | |
| app-id: ${{ secrets.DOCSY_BOT_APP_ID }} | |
| private-key: ${{ secrets.DOCSY_BOT_PRIVATE_KEY }} | |
| owner: unionai | |
| repositories: unionai-docs | |
| - name: Send repository_dispatch to unionai/unionai-docs | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: | | |
| if [ -z "${GH_TOKEN}" ]; then | |
| echo "::warning title=docs regen signal skipped::docsy-bot App not configured (DOCSY_BOT_APP_ID / DOCSY_BOT_PRIVATE_KEY) — skipping the unionai-docs regen signal. Set the secrets to enable it." | |
| exit 0 | |
| fi | |
| VERSION="${GITHUB_REF#refs/tags/v}" | |
| echo "Signaling unionai/unionai-docs to regenerate API docs for flyte ${VERSION}" | |
| gh api repos/unionai/unionai-docs/dispatches \ | |
| -f event_type=sdk-release \ | |
| -f 'client_payload[version]='"${VERSION}" |