Skip to content

Commit fec5182

Browse files
committed
ci: Record the OpenAPI spec version instead of committing the spec
1 parent fed06a3 commit fec5182

10 files changed

Lines changed: 212 additions & 29181 deletions

‎.gitattributes‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎.github/workflows/_checks.yaml‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -73,32 +73,6 @@ jobs:
7373
- name: Async docstrings check
7474
run: uv run poe check-docstrings
7575

76-
# Regenerates the models from the committed `spec/openapi.json` and fails when the result differs from what is
77-
# committed. Keeps the generated files a pure function of the snapshot plus the pinned codegen tooling, so a
78-
# codegen or formatter upgrade can't silently drift them, and nobody can hand-edit them.
79-
models_check:
80-
name: Models check
81-
runs-on: ubuntu-latest
82-
steps:
83-
- name: Checkout repository
84-
uses: actions/checkout@v7
85-
86-
- name: Set up Python
87-
uses: actions/setup-python@v7
88-
with:
89-
python-version: ${{ env.PYTHON_VERSION }}
90-
91-
- name: Set up uv package manager
92-
uses: astral-sh/setup-uv@v8.3.2
93-
with:
94-
python-version: ${{ env.PYTHON_VERSION }}
95-
96-
- name: Install dependencies
97-
run: uv run poe install-dev
98-
99-
- name: Generated models check
100-
run: uv run poe check-models
101-
10276
markdown_lint_check:
10377
name: Markdown lint check
10478
runs-on: ubuntu-latest

‎.github/workflows/on_schedule_regenerate_models.yaml‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# This workflow keeps the generated API models in sync with the published OpenAPI specification. Every night it
2-
# refreshes the committed `spec/openapi.json` snapshot, regenerates the models from it, and opens a pull request
3-
# when the result differs from master.
2+
# regenerates the models from the published specification and opens a pull request when the result differs from
3+
# master.
44
#
55
# Two invariants make it safe to run unattended:
66
# 1. Generation always happens on master, so the output is a function of the current spec and the current codegen
@@ -56,16 +56,14 @@ jobs:
5656
- name: Install dependencies
5757
run: uv run poe install-dev
5858

59-
- name: Sync the OpenAPI specification snapshot
60-
run: uv run poe update-spec
61-
59+
# Downloads the published specification, generates from it, and records its version in `pyproject.toml`.
6260
- name: Regenerate models
6361
run: uv run poe generate-models
6462

65-
# Gate on the generated models rather than on the specification: `info.version` in the published spec carries
66-
# a build timestamp that apify-docs bumps on every specification change, so a spec diff on its own says
67-
# nothing about whether the client is affected. As a result the committed snapshot only moves together with
68-
# the models it produces, which is exactly what `check-models` verifies.
63+
# Gate on the generated models rather than on the specification version: that version carries a build
64+
# timestamp that apify-docs bumps on every specification change, so a version bump on its own says nothing
65+
# about whether the client is affected. Because nothing is committed unless the models moved, the recorded
66+
# version follows the models and stays behind the published specification while the models are unaffected.
6967
- name: Check for model changes
7068
id: changes
7169
run: |
@@ -139,7 +137,7 @@ jobs:
139137
with:
140138
message: ${{ env.PR_TITLE }}
141139
add: >-
142-
spec/openapi.json
140+
pyproject.toml
143141
src/apify_client/_models.py
144142
src/apify_client/_typeddicts.py
145143
src/apify_client/_literals.py
@@ -153,10 +151,10 @@ jobs:
153151
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
154152
WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/workflows/on_schedule_regenerate_models.yaml
155153
run: |
156-
SPEC_VERSION=$(jq -r '.info.version' spec/openapi.json)
154+
SPEC_VERSION=$(uv run python -c 'import pathlib, tomllib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["tool"]["apify"]["openapi-spec"]["version"])')
157155
158156
BODY=$(printf '%s\n' \
159-
"- Syncs \`spec/openapi.json\` with the [published OpenAPI specification](https://docs.apify.com/api/openapi.json) and regenerates the Pydantic models, TypedDicts, and literal aliases from it." \
157+
"- Regenerates the Pydantic models, TypedDicts, and literal aliases from the [published OpenAPI specification](https://docs.apify.com/api/openapi.json), and records the version they were generated from in \`pyproject.toml\`." \
160158
"- Specification version: \`${SPEC_VERSION}\`." \
161159
"" \
162160
"> [!IMPORTANT]" \

‎.rules.md‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,15 @@ Uses [uv](https://docs.astral.sh/uv/) for project management and [Poe the Poet](
1212

1313
```bash
1414
uv run poe install-dev # Install dev deps + git hooks
15-
uv run poe check-code # Run all checks (lint, type-check, unit-tests, docstring + model checks)
15+
uv run poe check-code # Run all checks (lint, type-check, unit-tests, docstring check)
1616
uv run poe lint # Ruff format check + ruff check
1717
uv run poe format # Auto-fix lint issues and format
1818
uv run poe type-check # Run ty type checker
1919
uv run poe unit-tests # Run unit tests
2020
uv run poe check-docstrings # Verify async docstrings match sync
2121
uv run poe fix-docstrings # Auto-fix async docstrings
22-
uv run poe update-spec # Refresh spec/openapi.json from the published OpenAPI spec
23-
uv run poe generate-models # Regenerate _models.py, _typeddicts.py, _literals.py from spec/openapi.json
24-
uv run poe check-models # Verify the generated files match the committed spec
22+
uv run poe generate-models # Regenerate _models.py, _typeddicts.py, _literals.py from the published spec
23+
uv run poe generate-models-from-file <path> # Regenerate from a local OpenAPI spec file
2524

2625
# Run a single test
2726
uv run pytest tests/unit/test_file.py
@@ -78,14 +77,14 @@ Docstrings are written on sync clients and **automatically copied** to async cli
7877

7978
Each input-side TypedDict ships in two casings: snake_case (`RequestDict`) and camelCase (`RequestCamelDict`). The camel variants are synthesized in `scripts/postprocess_generated_models.py` from the Pydantic `Field(alias=...)` map; resource-client signatures union both with the Pydantic model so users can pass either casing without losing type-checker support.
8079

81-
`spec/openapi.json` is a **committed snapshot** of the published OpenAPI spec and the only input codegen reads. Generation is therefore a pure function of committed state — the same snapshot plus the same pinned tooling always produces the same output — which is what lets `check-models` verify the generated files offline.
82-
83-
- Generated by `datamodel-code-generator` from `spec/openapi.json` (config in `pyproject.toml` under `[tool.datamodel-codegen]`, aliases in `datamodel_codegen_aliases.json`)
80+
- Generated by `datamodel-code-generator` from the OpenAPI spec at `https://docs.apify.com/api/openapi.json` (config in `pyproject.toml` under `[tool.datamodel-codegen]`, aliases in `datamodel_codegen_aliases.json`)
8481
- After generation, `scripts/postprocess_generated_models.py` is run to apply additional fixes
85-
- To regenerate locally: `uv run poe update-spec` to refresh the snapshot, then `uv run poe generate-models`. To try a candidate spec, overwrite `spec/openapi.json` and regenerate
86-
- `uv run poe check-models` (part of `check-code`, and the `Models check` CI job) regenerates, compares, and restores the working tree — it never leaves the generated files modified, and it doesn't care whether they're committed. So a `datamodel-code-generator` or `ruff` upgrade that changes codegen output turns **its own** dependency-bump PR red — that PR carries the regenerated files, instead of the drift contaminating an unrelated spec change later
87-
- In CI, the `Regenerate models` workflow (`on_schedule_regenerate_models.yaml`) runs nightly at 02:00 UTC: it refreshes the snapshot, regenerates, and opens a PR when the models change. It always generates on master and rebuilds its `ci/regenerate-models` branch from master instead of appending, so the PR diff is always "current spec vs current master". The PR opens with a mergeable `chore:` title — retitle it to `fix:`/`feat:` when the diff is user-facing, so it lands in the changelog and triggers a release
88-
- The gate is the **generated models**, not the spec: `info.version` in the published spec carries a build timestamp that apify-docs bumps on every spec change, so the snapshot only moves together with the models it produces
82+
- To regenerate locally:
83+
- From the live published spec: `uv run poe generate-models`
84+
- From a local spec file: `uv run poe generate-models-from-file path/to/openapi.json`
85+
- `scripts/openapi_spec.py` downloads the spec once into git-ignored `tmp/openapi.json` (both codegen passes read that copy, so a spec redeployed mid-run can't leave the generated files describing two different inputs), and afterwards records the spec's `info.version` in `pyproject.toml` under `[tool.apify.openapi-spec]`. That version is **provenance only** — the spec is served latest-only, so it can't fetch that exact document back. It moves together with the generated files, so it stays behind the published spec whenever a spec change leaves the models unchanged
86+
- In CI, the `Regenerate models` workflow (`on_schedule_regenerate_models.yaml`) runs nightly at 02:00 UTC: it regenerates from the published spec and opens a PR when the models change. It always generates on master and rebuilds its `ci/regenerate-models` branch from master instead of appending, so the PR diff is always "current spec vs current master". The PR opens with a mergeable `chore:` title — retitle it to `fix:`/`feat:` when the diff is user-facing, so it lands in the changelog and triggers a release
87+
- The gate is the **generated models**, not the spec version: that version carries a build timestamp apify-docs bumps on every spec change, so a version bump on its own says nothing about whether the client is affected
8988
- Manual regeneration is also possible from the GitHub Actions UI (`Regenerate models` workflow)
9089

9190
## Code Conventions

‎pyproject.toml‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,15 @@ custom_file_header = "# generated by datamodel-codegen"
246246
disable_timestamp = true
247247
keep_model_order = true
248248

249+
# Which published OpenAPI specification the committed `_models.py`, `_typeddicts.py`, and `_literals.py` were
250+
# generated from. `poe generate-models` records it; `apify/apify-docs` bumps it on every specification change
251+
# (`apify-api/openapi/openapi.yaml` reads it from `components/version.yaml`). This is provenance only - the
252+
# specification is served latest-only, so the version cannot fetch that exact document back. It therefore moves
253+
# together with the generated files and stays behind the published specification whenever a specification change
254+
# leaves the models unchanged.
255+
[tool.apify.openapi-spec]
256+
version = "v2-2026-07-28T083939Z"
257+
249258
[tool.uv]
250259
# Minimal defense against supply-chain atatcks.
251260
exclude-newer = "24 hours"
@@ -270,10 +279,7 @@ integration-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tes
270279
integration-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify_client --cov-report=xml:coverage-integration.xml tests/integration"
271280
check-docstrings = "uv run python -m scripts.check_docstrings"
272281
fix-docstrings = "uv run python -m scripts.fix_docstrings"
273-
update-spec = "uv run python -m scripts.update_openapi_spec"
274-
# Regenerates to compare, then restores the working tree - running it never modifies the generated files.
275-
check-models = "uv run python -m scripts.check_generated_models"
276-
check-code = ["lint", "type-check", "check-docstrings", "check-models", "unit-tests"]
282+
check-code = ["lint", "type-check", "check-docstrings", "unit-tests"]
277283

278284
[tool.poe.tasks.install-dev]
279285
shell = "uv sync --all-extras && uv run pre-commit install"
@@ -296,18 +302,35 @@ cwd = "website"
296302
shell = "./build_api_reference.sh && pnpm install && uv run pnpm start"
297303
cwd = "website"
298304

299-
# Generated from the committed `spec/openapi.json` snapshot rather than the published URL, so the output depends
300-
# only on committed state and `check-models` can verify it offline. Run `poe update-spec` to refresh the snapshot.
305+
# Both codegen passes read one downloaded copy of the specification, so a specification redeployed mid-run cannot
306+
# leave the generated files describing two different inputs. The version is recorded last, once generation
307+
# succeeded, so it always names the specification the generated files follow from.
301308
#
302309
# The `--alias-generator to_camel` flag lives on the `_models.py` command (not in `[tool.datamodel-codegen]`)
303310
# because datamodel-codegen only allows it for `pydantic_v2.BaseModel` output and would reject the TypedDict run.
304311
[tool.poe.tasks.generate-models]
305312
shell = """
306-
uv run datamodel-codegen --input spec/openapi.json \
313+
uv run python -m scripts.openapi_spec fetch \
314+
&& uv run datamodel-codegen --input tmp/openapi.json \
315+
--alias-generator to_camel \
316+
&& uv run datamodel-codegen --input tmp/openapi.json \
317+
--output src/apify_client/_typeddicts.py \
318+
--output-model-type typing.TypedDict \
319+
--no-use-closed-typed-dict \
320+
&& python scripts/postprocess_generated_models.py \
321+
&& uv run python -m scripts.openapi_spec record-version
322+
"""
323+
324+
# A candidate specification that isn't published yet. The recorded version is deliberately left alone - it names
325+
# the published specification the committed models follow from.
326+
[tool.poe.tasks.generate-models-from-file]
327+
shell = """
328+
uv run datamodel-codegen --input $input_file \
307329
--alias-generator to_camel \
308-
&& uv run datamodel-codegen --input spec/openapi.json \
330+
&& uv run datamodel-codegen --input $input_file \
309331
--output src/apify_client/_typeddicts.py \
310332
--output-model-type typing.TypedDict \
311333
--no-use-closed-typed-dict \
312334
&& python scripts/postprocess_generated_models.py
313335
"""
336+
args = [{ name = "input-file", positional = true, required = true }]

‎scripts/check_generated_models.py‎

Lines changed: 0 additions & 66 deletions
This file was deleted.

0 commit comments

Comments
 (0)