Skip to content

Commit 7b1310e

Browse files
committed
ci: Address code review on the OpenAPI spec version recording
1 parent fec5182 commit 7b1310e

5 files changed

Lines changed: 313 additions & 51 deletions

File tree

‎.github/workflows/on_schedule_regenerate_models.yaml‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -56,22 +56,30 @@ jobs:
5656
- name: Install dependencies
5757
run: uv run poe install-dev
5858

59+
# Read before regenerating overwrites it, so the pull request can say whether the specification moved at all.
60+
- name: Read the recorded specification version
61+
id: previous-spec
62+
run: echo "version=$(uv run python -m scripts.openapi_spec recorded-version)" >> "$GITHUB_OUTPUT"
63+
5964
# Downloads the published specification, generates from it, and records its version in `pyproject.toml`.
6065
- name: Regenerate models
6166
run: uv run poe generate-models
6267

63-
# Gate on the generated models rather than on the specification version: that version carries a build
64-
# timestamp that apify-docs bumps on every specification change, so a version bump on its own says nothing
65-
# about whether the client is affected. Because nothing is committed unless the models moved, the recorded
66-
# version follows the models and stays behind the published specification while the models are unaffected.
68+
# Gate on the generated models rather than on the specification version: that version is a build timestamp
69+
# that moves on rebuilds regardless of client impact, and doesn't reliably move at all when the
70+
# specification content does (see the pull request body step below). Because nothing is committed unless the
71+
# models moved, the recorded version follows the models rather than the published specification.
72+
#
73+
# Compared against HEAD rather than the index, so the result doesn't depend on nothing having been staged
74+
# earlier in the job.
6775
- name: Check for model changes
6876
id: changes
6977
run: |
70-
if git diff --quiet -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py; then
78+
if git diff --quiet HEAD -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py; then
7179
echo "Models are already up to date with the published specification."
7280
echo "has-changes=false" >> "$GITHUB_OUTPUT"
7381
else
74-
git diff --stat -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py
82+
git diff --stat HEAD -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py
7583
echo "has-changes=true" >> "$GITHUB_OUTPUT"
7684
fi
7785
@@ -85,17 +93,31 @@ jobs:
8593
env:
8694
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
8795
run: |
88-
if ! git ls-remote --exit-code --heads origin "$BRANCH_NAME" >/dev/null 2>&1; then
96+
# `--exit-code` reports 2 for "no such branch", so anything else is a real failure and must not be read
97+
# as an absent branch - that would retire a current pull request over a transient network error.
98+
git ls-remote --exit-code --heads origin "$BRANCH_NAME" >/dev/null 2>&1 && status=0 || status=$?
99+
if [[ "$status" == "2" ]]; then
89100
echo "No auto-update branch exists yet."
90101
echo "is-new=true" >> "$GITHUB_OUTPUT"
91102
exit 0
103+
elif [[ "$status" != "0" ]]; then
104+
echo "Failed to look up branch $BRANCH_NAME on the remote (git ls-remote exited $status)." >&2
105+
exit 1
92106
fi
93107
94108
# The branch only counts as "already up for review" while a pull request is actually open on it. A
95109
# leftover branch - an earlier run that failed before opening one, or a review that closed the pull
96110
# request without deleting the branch - must be rebuilt, otherwise its contents would look proposed
97111
# and silently suppress every future regeneration.
98-
if [[ -z "$(gh pr list --head "$BRANCH_NAME" --base master --state open --json number --jq '.[0].number // empty')" ]]; then
112+
#
113+
# An empty result only counts once the query itself succeeded: `gh` exits non-zero on an API error, and
114+
# treating that as "no open pull request" would replace a pull request someone may be reviewing.
115+
if ! PR_NUMBER=$(gh pr list --head "$BRANCH_NAME" --base master --state open --json number --jq '.[0].number // empty'); then
116+
echo "Failed to list open pull requests for $BRANCH_NAME." >&2
117+
exit 1
118+
fi
119+
120+
if [[ -z "$PR_NUMBER" ]]; then
99121
echo "Branch $BRANCH_NAME has no open pull request - rebuilding it."
100122
echo "is-new=true" >> "$GITHUB_OUTPUT"
101123
exit 0
@@ -150,12 +172,23 @@ jobs:
150172
env:
151173
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
152174
WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/workflows/on_schedule_regenerate_models.yaml
175+
PREVIOUS_SPEC_VERSION: ${{ steps.previous-spec.outputs.version }}
153176
run: |
154-
SPEC_VERSION=$(uv run python -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["tool"]["apify"]["openapi-spec"]["version"])')
177+
SPEC_VERSION=$(uv run python -m scripts.openapi_spec recorded-version)
178+
179+
# A moved version stamp proves the specification changed. An unchanged one proves nothing: apify-docs
180+
# bumps `components/version.yaml` in a follow-up commit marked `[skip ci]`, so a deploy publishes new
181+
# specification content still carrying the previous stamp. Say which case this is rather than let the
182+
# reviewer assume the stamp is a content identity.
183+
if [[ "$SPEC_VERSION" == "$PREVIOUS_SPEC_VERSION" ]]; then
184+
SPEC_LINE="- Specification version: \`${SPEC_VERSION}\` - **unchanged**. The published stamp lags its content by a deploy, so this diff is either an unstamped specification change or a codegen/formatter change - the diff itself is the only way to tell."
185+
else
186+
SPEC_LINE="- Specification version: \`${PREVIOUS_SPEC_VERSION}\` -> \`${SPEC_VERSION}\`."
187+
fi
155188
156189
BODY=$(printf '%s\n' \
157190
"- Regenerates the Pydantic models, TypedDicts, and literal aliases from the [published OpenAPI specification](https://docs.apify.com/api/openapi.json), and records the version they were generated from in \`pyproject.toml\`." \
158-
"- Specification version: \`${SPEC_VERSION}\`." \
191+
"${SPEC_LINE}" \
159192
"" \
160193
"> [!IMPORTANT]" \
161194
"> Retitle this pull request to \`fix:\` or \`feat:\` when the diff is user-facing, so that it lands in the changelog and triggers a release - \`chore:\` does neither." \

‎.rules.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,11 @@ Each input-side TypedDict ships in two casings: snake_case (`RequestDict`) and c
8282
- To regenerate locally:
8383
- From the live published spec: `uv run poe generate-models`
8484
- From a local spec file: `uv run poe generate-models-from-file path/to/openapi.json`
85-
- `scripts/openapi_spec.py` downloads the spec once into git-ignored `tmp/openapi.json` (both codegen passes read that copy, so a spec redeployed mid-run can't leave the generated files describing two different inputs), and afterwards records the spec's `info.version` in `pyproject.toml` under `[tool.apify.openapi-spec]`. That version is **provenance only** — the spec is served latest-only, so it can't fetch that exact document back. It moves together with the generated files, so it stays behind the published spec whenever a spec change leaves the models unchanged
85+
- `scripts/openapi_spec.py` downloads the spec once into git-ignored `tmp/openapi.json` (both codegen passes read that copy, so a spec redeployed mid-run can't leave the generated files describing two different inputs), and afterwards records the spec's `info.version` in `pyproject.toml` under `[tool.apify.openapi-spec]`
86+
- That version is a **coarse marker, not a content identity**: the spec is served latest-only, so it can't fetch that exact document back, and apify-docs bumps `components/version.yaml` in a follow-up `[skip ci]` commit — so a deploy publishes new spec content still carrying the previous stamp. A moved value proves the spec changed; an unchanged value proves nothing
87+
- The recording is unconditional; the nightly workflow is what keeps it honest, by committing nothing unless the models changed. So the recorded version lands together with the generated files and stays behind the published spec while spec changes leave the models unchanged. A local `generate-models` that only moves that line is the same case — drop it instead of committing it alone. `generate-models-from-file` deliberately leaves the line alone, so models generated from a candidate spec must not be committed
8688
- In CI, the `Regenerate models` workflow (`on_schedule_regenerate_models.yaml`) runs nightly at 02:00 UTC: it regenerates from the published spec and opens a PR when the models change. It always generates on master and rebuilds its `ci/regenerate-models` branch from master instead of appending, so the PR diff is always "current spec vs current master". The PR opens with a mergeable `chore:` title — retitle it to `fix:`/`feat:` when the diff is user-facing, so it lands in the changelog and triggers a release
87-
- The gate is the **generated models**, not the spec version: that version carries a build timestamp apify-docs bumps on every spec change, so a version bump on its own says nothing about whether the client is affected
89+
- The gate is the **generated models**, not the spec version: the version is a build timestamp that moves on spec rebuilds regardless of client impact, and (per above) doesn't reliably move at all — so it can't decide whether to open a PR
8890
- Manual regeneration is also possible from the GitHub Actions UI (`Regenerate models` workflow)
8991

9092
## Code Conventions

‎pyproject.toml‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -247,11 +247,17 @@ disable_timestamp = true
247247
keep_model_order = true
248248

249249
# Which published OpenAPI specification the committed `_models.py`, `_typeddicts.py`, and `_literals.py` were
250-
# generated from. `poe generate-models` records it; `apify/apify-docs` bumps it on every specification change
251-
# (`apify-api/openapi/openapi.yaml` reads it from `components/version.yaml`). This is provenance only - the
252-
# specification is served latest-only, so the version cannot fetch that exact document back. It therefore moves
253-
# together with the generated files and stays behind the published specification whenever a specification change
254-
# leaves the models unchanged.
250+
# generated from. This is a coarse marker, not a content identity: the specification is served latest-only, so the
251+
# version cannot fetch that exact document back, and `apify/apify-docs` bumps
252+
# `apify-api/openapi/components/version.yaml` in a follow-up commit marked `[skip ci]` - a deploy therefore
253+
# publishes new specification content that still carries the previous stamp. A moved value proves the
254+
# specification changed; an unchanged one proves nothing.
255+
#
256+
# `poe generate-models` records whatever it downloaded, unconditionally. What keeps the recorded version honest is
257+
# the nightly workflow committing nothing unless the models changed: the value therefore lands together with the
258+
# generated files, and stays behind the published specification while specification changes leave the models
259+
# unchanged. A local regeneration that only moves this line is that same case - drop the change rather than
260+
# committing it on its own.
255261
[tool.apify.openapi-spec]
256262
version = "v2-2026-07-28T083939Z"
257263

@@ -321,8 +327,9 @@ uv run python -m scripts.openapi_spec fetch \
321327
&& uv run python -m scripts.openapi_spec record-version
322328
"""
323329

324-
# A candidate specification that isn't published yet. The recorded version is deliberately left alone - it names
325-
# the published specification the committed models follow from.
330+
# A candidate specification that isn't published yet, so the recorded version is deliberately left alone - it
331+
# names the published specification. Committing models generated this way would make it name a document they did
332+
# not come from, hence the warning.
326333
[tool.poe.tasks.generate-models-from-file]
327334
shell = """
328335
uv run datamodel-codegen --input $input_file \
@@ -331,6 +338,7 @@ uv run datamodel-codegen --input $input_file \
331338
--output src/apify_client/_typeddicts.py \
332339
--output-model-type typing.TypedDict \
333340
--no-use-closed-typed-dict \
334-
&& python scripts/postprocess_generated_models.py
341+
&& python scripts/postprocess_generated_models.py \
342+
&& echo "Generated from $input_file - the version in [tool.apify.openapi-spec] still names the published specification, so do not commit these models."
335343
"""
336344
args = [{ name = "input-file", positional = true, required = true }]

0 commit comments

Comments
 (0)