Claim new flyteplugins PyPI names at merge time, not release time #1

Workflow file for this run

.github/workflows/claim-plugin-names.yml at d10b51e

	name: Claim plugin names

	# PyPI limits each account to creating 4 new projects per 24 hours
	# (https://github.com/pypi/support/issues/10572). Uploads to projects that
	# already exist are unlimited. A flyteplugins name that sits declared in the
	# tree but unpublished is also squattable by anyone (see the
	# discover-plugin-packages comment in publish.yml -- four names were lost
	# exactly that way). This workflow therefore:
	#
	# * on pull requests touching plugin packaging: fails early when a single
	# PR adds more than 4 new flyteplugins distributions, so the limit is
	# surfaced to the author before merge instead of breaking the release;
	# * on merge to main: immediately publishes a 0.0.0a0 wheel for every
	# still-unregistered flyteplugins name, claiming the PyPI project the
	# day the plugin lands rather than at the next release;
	# * daily: drains any backlog (names that could not be claimed yet, e.g.
	# after PyPI frees a quarantined name) at the allowed 4 per day.
	#
	# Net effect: release runs of publish.yml only ever upload to existing
	# projects and never hit the new-project rate limit.

	on:
	pull_request:
	paths:
	- "plugins/**/pyproject.toml"
	- ".github/workflows/claim-plugin-names.yml"
	push:
	branches: [main]
	paths:
	- "plugins/**/pyproject.toml"
	schedule:
	- cron: "23 14 * * *"
	workflow_dispatch:

	env:
	# PyPI's new-project creation limit per account per 24h.
	MAX_NEW_PROJECTS: 4

	concurrency:
	group: claim-plugin-names-${{ github.event_name == 'pull_request' && github.ref \|\| 'main' }}
	cancel-in-progress: false

	jobs:
	check-pr-plugin-count:
	name: PR adds at most 4 new plugin distributions
	if: github.event_name == 'pull_request'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	with:
	fetch-depth: "0"
	- name: Count flyteplugins distributions added by this PR
	run: \|
	set -euo pipefail
	BASE=$(git merge-base "origin/${GITHUB_BASE_REF}" HEAD)

	# All flyteplugins distribution names declared at a given revision.
	# NF <= 4 mirrors publish.yml's `find -maxdepth 3` (plugins/<x> and
	# plugins/agents/<x>).
	names_at() {
	git ls-tree -r --name-only "$1" -- plugins \
	\| awk -F/ '$NF == "pyproject.toml" && NF <= 4' \
	\| while read -r f; do
	git show "$1:$f" \
	\| sed -nE 's/^name[[:space:]]=[[:space:]]"(flyteplugins[^"])"./\1/p' \
	\| head -n1
	done \| sort -u
	}

	HEAD_NAMES=$(names_at HEAD)
	BASE_NAMES=$(names_at "$BASE")
	NEW=$(comm -13 <(printf '%s\n' "$BASE_NAMES") <(printf '%s\n' "$HEAD_NAMES") \| sed '/^$/d')
	COUNT=$(printf '%s' "$NEW" \| grep -c . \|\| true)

	if [ "$COUNT" -eq 0 ]; then
	echo "This PR adds no new flyteplugins distributions."
	exit 0
	fi

	echo "This PR adds $COUNT new flyteplugins distribution(s):"
	printf '%s\n' "$NEW"

	if [ "$COUNT" -gt "$MAX_NEW_PROJECTS" ]; then
	{
	echo "## Too many new plugin packages in one PR"
	echo
	echo "This PR adds $COUNT new \`flyteplugins-*\` distributions:"
	echo
	printf '%s\n' "$NEW" \| sed 's/^/- `/;s/$/`/'
	echo
	echo "PyPI limits each account to creating $MAX_NEW_PROJECTS new projects per 24 hours"
	echo "(https://github.com/pypi/support/issues/10572), and every new plugin name must be"
	echo "registered on PyPI the day it merges -- an unregistered name can be squatted by"
	echo "anyone. Please split this PR so that no single PR adds more than"
	echo "$MAX_NEW_PROJECTS new plugin distributions."
	} >> "$GITHUB_STEP_SUMMARY"
	echo "::error::This PR adds $COUNT new flyteplugins distributions, but PyPI only allows $MAX_NEW_PROJECTS new projects per account per 24h (https://github.com/pypi/support/issues/10572). Split it into PRs adding at most $MAX_NEW_PROJECTS new plugin distributions each."
	exit 1
	fi

	echo "Within the PyPI new-project budget ($MAX_NEW_PROJECTS/24h); the names will be claimed on merge."

	claim:
	name: Claim unregistered plugin names on PyPI
	if: github.event_name != 'pull_request' && github.repository == 'flyteorg/flyte-sdk'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	with:
	fetch-depth: "0"
	- name: Discover unregistered flyteplugins names
	id: discover
	run: \|
	set -euo pipefail
	: > unclaimed.txt

	files=$(find plugins -mindepth 1 -maxdepth 3 -name pyproject.toml -not -path '/.venv/' \
	-exec grep -lE '^name[[:space:]]=[[:space:]]"flyteplugins' {} +)

	for f in $files; do
	name=$(sed -nE 's/^name[[:space:]]=[[:space:]]"([^"]+)".*/\1/p' "$f" \| head -n1)
	norm=$(echo "$name" \| tr '[:upper:]' '[:lower:]' \| sed -E 's/[-_.]+/-/g')
	code=$(curl -sS -o /dev/null -w '%{http_code}' --retry 3 "https://pypi.org/simple/${norm}/")
	case "$code" in
	200) echo "already on PyPI: $name" ;;
	404) printf '%s\t%s\n' "$name" "$(dirname "$f")" >> unclaimed.txt ;;
	*)
	echo "::error::Unexpected HTTP $code from https://pypi.org/simple/${norm}/ while checking ${name}; refusing to guess."
	exit 1
	;;
	esac
	done

	COUNT=$(grep -c . unclaimed.txt \|\| true)
	echo "count=$COUNT" >> "$GITHUB_OUTPUT"

	if [ "$COUNT" -eq 0 ]; then
	echo "Every flyteplugins distribution is already registered on PyPI; nothing to claim."
	exit 0
	fi

	echo "Unregistered on PyPI ($COUNT):"
	cut -f1 unclaimed.txt

	# A merge should never surface more than one day's budget: the PR
	# check above enforces <= MAX_NEW_PROJECTS per PR, so more than
	# that on a push means the check was bypassed. Fail loudly before
	# publishing anything so a human splits the work up.
	if [ "$COUNT" -gt "$MAX_NEW_PROJECTS" ] && [ "${GITHUB_EVENT_NAME}" = "push" ]; then
	echo "::error::$COUNT flyteplugins names are unregistered on PyPI, but PyPI only allows $MAX_NEW_PROJECTS new projects per account per 24h (https://github.com/pypi/support/issues/10572). The offending PR should be reverted and split into PRs adding at most $MAX_NEW_PROJECTS new plugin distributions; the daily scheduled run of this workflow will otherwise drain the backlog at $MAX_NEW_PROJECTS per day."
	exit 1
	fi

	# Scheduled/manual runs drain a large backlog within the limit
	# instead of failing, claiming the first MAX_NEW_PROJECTS today.
	if [ "$COUNT" -gt "$MAX_NEW_PROJECTS" ]; then
	echo "::warning::$COUNT names are unregistered but PyPI allows only $MAX_NEW_PROJECTS new projects per 24h; claiming the first $MAX_NEW_PROJECTS now, the rest on subsequent daily runs."
	head -n "$MAX_NEW_PROJECTS" unclaimed.txt > today.txt
	mv today.txt unclaimed.txt
	fi
	- name: Set up Python
	if: steps.discover.outputs.count != '0'
	uses: actions/setup-python@v7
	with:
	python-version: "3.13"
	- name: Install uv
	if: steps.discover.outputs.count != '0'
	uses: astral-sh/setup-uv@v10.0.0
	- name: Build and publish claim wheels
	if: steps.discover.outputs.count != '0'
	env:
	# 0.0.0a0 is a pre-release, so default pip resolution never picks
	# the claim wheel over a real release; it exists only to register
	# the project name under the flyte-bot account. Pinning also keeps
	# setuptools_scm from emitting a PEP 440 local version (+g<sha>)
	# that PyPI rejects (same reasoning as in publish.yml).
	SETUPTOOLS_SCM_PRETEND_VERSION: 0.0.0a0
	TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
	TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
	run: \|
	set -euo pipefail
	while IFS=$'\t' read -r name dir; do
	echo "::group::Claiming $name from $dir"
	(
	cd "$dir"
	uv venv
	uv pip install build twine setuptools wheel
	uv run python -m build --wheel
	uvx twine upload --verbose --skip-existing dist/*.whl
	)
	echo "::endgroup::"
	done < unclaimed.txt

Navigation Menu

Uh oh!

Claim new flyteplugins PyPI names at merge time, not release time #1

Workflow file

Claim new flyteplugins PyPI names at merge time, not release time #1

Uh oh!

Workflow file for this run