Skip to content

Commit 842c82d

Browse files
vdusekclaude
andcommitted
refactor: move generated literals to _literals_generated.py and snake_case API values
Split the 11 generated `Literal` type aliases out of `_models_generated.py` into a new `_literals_generated.py` so downstream code can depend on the literals without pulling in every Pydantic model. Rename the hand-maintained `_types.py` to `_literals.py` to match. Convert camelCase literal values (`'ownedByMe'`, `'sharedWithMe'`) to snake_case and emit a wire-format mapping so the API still receives the camelCase form it expects. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 003986b commit 842c82d

44 files changed

Lines changed: 964 additions & 588 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/02_concepts/code/03_nested_async.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from apify_client import ApifyClientAsync
2-
from apify_client._models_generated import ActorJobStatus
2+
from apify_client._literals_generated import ActorJobStatus
33

44
TOKEN = 'MY-APIFY-TOKEN'
55

‎docs/02_concepts/code/03_nested_sync.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from apify_client import ApifyClient
2-
from apify_client._models_generated import ActorJobStatus
2+
from apify_client._literals_generated import ActorJobStatus
33

44
TOKEN = 'MY-APIFY-TOKEN'
55

‎pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ indent-style = "space"
123123
"**/__init__.py" = [
124124
"F401", # Unused imports
125125
]
126+
"**/{_models,_models_generated}.py" = [
127+
"TC001", # Pydantic needs the literal aliases importable at runtime to resolve forward references
128+
]
126129
"**/{scripts}/*" = [
127130
"D", # Everything from the pydocstyle
128131
"INP001", # File {filename} is part of an implicit namespace package, add an __init__.py

‎scripts/postprocess_generated_models.py‎

Lines changed: 167 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@
66
- Deduplicate the inlined `Type(StrEnum)` that comes from ErrorResponse.yaml; rewire to `ErrorType`.
77
- Rewrite every `class X(StrEnum)` as `X: TypeAlias = Literal[...]` so downstream code can pass
88
plain strings (and reuse the named alias in resource-client signatures) instead of enum members.
9+
- Convert camelCase string values in each literal alias to snake_case (Pythonic), and emit a
10+
`_<NAME>_WIRE_VALUES` mapping the Python value back to the original camelCase form so the
11+
resource clients can still produce the exact string the API expects on the wire.
12+
- Move the resulting `TypeAlias = Literal[...]` definitions into `_literals_generated.py`, leaving
13+
`_models_generated.py` importing them — so consumers can depend on a dedicated literals module
14+
without pulling in every Pydantic model.
915
- Add `@docs_group('Models')` to every model class (plus the required import).
1016
1117
Applied to `_typeddicts_generated.py`:
@@ -30,6 +36,7 @@
3036
REPO_ROOT = Path(__file__).resolve().parent.parent
3137
PACKAGE_DIR = REPO_ROOT / 'src' / 'apify_client'
3238
MODELS_PATH = PACKAGE_DIR / '_models_generated.py'
39+
LITERALS_PATH = PACKAGE_DIR / '_literals_generated.py'
3340
TYPEDDICTS_PATH = PACKAGE_DIR / '_typeddicts_generated.py'
3441

3542
# Map of camelCase discriminator values to their snake_case equivalents.
@@ -156,6 +163,141 @@ def convert_enums_to_literals(content: str) -> str:
156163
return _ensure_typing_import(_collapse_blank_lines('\n'.join(lines)), 'TypeAlias')
157164

158165

166+
LITERALS_FILE_HEADER = """\
167+
# generated by postprocess_generated_models
168+
169+
from __future__ import annotations
170+
171+
from typing import Literal, TypeAlias
172+
173+
174+
"""
175+
176+
_CAMEL_CASE_VALUE = re.compile(r"^'([a-z][a-zA-Z0-9]*[A-Z][a-zA-Z0-9]*)',?$")
177+
178+
179+
def _camel_to_snake(value: str) -> str:
180+
"""Convert a camelCase identifier to snake_case."""
181+
return re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', value).lower()
182+
183+
184+
def _is_literal_alias(node: ast.stmt) -> bool:
185+
"""Return True if `node` is a top-level `Name: TypeAlias = Literal[...]` statement."""
186+
return (
187+
isinstance(node, ast.AnnAssign)
188+
and isinstance(node.target, ast.Name)
189+
and isinstance(node.annotation, ast.Name)
190+
and node.annotation.id == 'TypeAlias'
191+
and isinstance(node.value, ast.Subscript)
192+
and isinstance(node.value.value, ast.Name)
193+
and node.value.value.id == 'Literal'
194+
)
195+
196+
197+
def snake_case_camelcase_literal_values(content: str) -> str:
198+
"""Rewrite camelCase string values in `Literal[...]` aliases into snake_case.
199+
200+
Scans each `Name: TypeAlias = Literal[...]` block and, for any value matching the camelCase
201+
pattern (lowercase-first followed by an uppercase letter), converts it to snake_case. For each
202+
alias that had at least one conversion, emits a `_<NAME>_WIRE_VALUES: dict[<NAME>, str] = ...`
203+
mapping right after the alias so consumers can translate back to the API wire format.
204+
205+
SCREAMING_SNAKE_CASE, dotted, hyphenated, and HTTP-method values pass through unchanged.
206+
"""
207+
tree = ast.parse(content)
208+
lines = content.split('\n')
209+
insertions: list[tuple[int, list[str]]] = [] # (insert-after-line-exclusive, lines to insert)
210+
211+
for alias_name, node, end_line in _extract_top_level_symbols(tree):
212+
if not _is_literal_alias(node):
213+
continue
214+
215+
assert node.end_lineno is not None # noqa: S101
216+
wire_mapping: dict[str, str] = {}
217+
for line_idx in range(node.lineno - 1, node.end_lineno):
218+
match = _CAMEL_CASE_VALUE.match(lines[line_idx].strip())
219+
if match is None:
220+
continue
221+
original = match.group(1)
222+
snake = _camel_to_snake(original)
223+
wire_mapping[snake] = original
224+
lines[line_idx] = lines[line_idx].replace(f"'{original}'", f"'{snake}'", 1)
225+
226+
if not wire_mapping:
227+
continue
228+
229+
constant_name = '_' + _camel_to_snake(alias_name).upper() + '_WIRE_VALUES'
230+
docstring = f'"""Maps snake_case `{alias_name}` values to the camelCase form expected on the API wire."""'
231+
mapping_lines = [
232+
'',
233+
f'{constant_name}: dict[{alias_name}, str] = {{',
234+
*(f" '{snake}': '{original}'," for snake, original in wire_mapping.items()),
235+
'}',
236+
docstring,
237+
]
238+
# Insert after the alias's trailing docstring (absorbed into end_line) so the docstring
239+
# stays attached to the alias rather than to the mapping dict.
240+
insertions.append((end_line, mapping_lines))
241+
242+
if not insertions:
243+
return content
244+
245+
for insert_at, new_lines in sorted(insertions, key=lambda r: r[0], reverse=True):
246+
lines[insert_at:insert_at] = new_lines
247+
248+
return '\n'.join(lines)
249+
250+
251+
def split_literals_to_file(content: str) -> tuple[str, str]:
252+
"""Move every top-level `Name: TypeAlias = Literal[...]` block into a separate literals module.
253+
254+
Walks the top-level AST, collects each literal alias plus its trailing bare-string docstring,
255+
deletes them from `_models_generated.py`, and rebuilds `_literals_generated.py` from the blocks
256+
in original order. The models content gains a `from apify_client._literals_generated import ...`
257+
line so Pydantic can still resolve the forward references in field annotations.
258+
259+
Returns `(new_models_content, literals_file_content)`. If no literal aliases are found, the
260+
models content is returned unchanged and the literals content is empty.
261+
"""
262+
tree = ast.parse(content)
263+
lines = content.split('\n')
264+
265+
blocks: list[tuple[int, int, str]] = [
266+
(node.lineno - 1, end_line, name)
267+
for name, node, end_line in _extract_top_level_symbols(tree)
268+
if _is_literal_alias(node)
269+
]
270+
271+
if not blocks:
272+
return content, ''
273+
274+
literal_lines: list[str] = []
275+
for start, end, _ in blocks:
276+
literal_lines.extend(lines[start:end])
277+
literal_lines.append('')
278+
literal_lines.append('')
279+
280+
new_lines = lines[:]
281+
for start, end, _ in sorted(blocks, key=lambda b: b[0], reverse=True):
282+
del new_lines[start:end]
283+
284+
# Inject the import right after the last existing `from apify_client.` import so ruff/isort
285+
# keep the final ordering stable.
286+
names = sorted(name for _, _, name in blocks)
287+
import_line = f'from apify_client._literals_generated import {", ".join(names)}'
288+
insert_at = next(
289+
(idx + 1 for idx in range(len(new_lines) - 1, -1, -1) if new_lines[idx].startswith('from apify_client.')),
290+
None,
291+
)
292+
if insert_at is None:
293+
raise RuntimeError('No `from apify_client.` import found in generated models to anchor literals import')
294+
new_lines.insert(insert_at, import_line)
295+
296+
models_content = _collapse_blank_lines('\n'.join(new_lines))
297+
literals_content = _collapse_blank_lines(LITERALS_FILE_HEADER + '\n'.join(literal_lines))
298+
return models_content, literals_content
299+
300+
159301
def add_docs_group_decorators(content: str, group_name: GroupName) -> str:
160302
"""Add `@docs_group(group_name)` to every class and inject the required import.
161303
@@ -319,17 +461,30 @@ def rename_with_dict_suffix(content: str, names: set[str]) -> str:
319461
return content
320462

321463

322-
def postprocess_models(path: Path) -> bool:
323-
"""Apply `_models_generated.py`-specific fixes. Returns True if the file changed."""
324-
original = path.read_text()
464+
def postprocess_models(models_path: Path, literals_path: Path) -> list[Path]:
465+
"""Apply `_models_generated.py`-specific fixes and emit `_literals_generated.py`.
466+
467+
Returns the list of paths that were (re)written.
468+
"""
469+
original = models_path.read_text()
325470
fixed = fix_discriminators(original)
326471
fixed = deduplicate_error_type_enum(fixed)
327472
fixed = convert_enums_to_literals(fixed)
328473
fixed = add_docs_group_decorators(fixed, 'Models')
329-
if fixed == original:
330-
return False
331-
path.write_text(fixed)
332-
return True
474+
models_content, literals_content = split_literals_to_file(fixed)
475+
if literals_content:
476+
literals_content = snake_case_camelcase_literal_values(literals_content)
477+
478+
changed: list[Path] = []
479+
if models_content != original:
480+
models_path.write_text(models_content)
481+
changed.append(models_path)
482+
if literals_content:
483+
previous = literals_path.read_text() if literals_path.exists() else ''
484+
if literals_content != previous:
485+
literals_path.write_text(literals_content)
486+
changed.append(literals_path)
487+
return changed
333488

334489

335490
def postprocess_typeddicts(path: Path) -> bool:
@@ -353,12 +508,12 @@ def run_ruff(paths: list[Path]) -> None:
353508

354509

355510
def main() -> None:
356-
changed: list[Path] = []
357-
if postprocess_models(MODELS_PATH):
358-
changed.append(MODELS_PATH)
359-
print(f'Fixed generated models in {MODELS_PATH}')
511+
changed = postprocess_models(MODELS_PATH, LITERALS_PATH)
512+
if changed:
513+
for path in changed:
514+
print(f'Wrote {path}')
360515
else:
361-
print('No fixes needed for _models_generated.py')
516+
print('No fixes needed for _models_generated.py / _literals_generated.py')
362517

363518
if postprocess_typeddicts(TYPEDDICTS_PATH):
364519
changed.append(TYPEDDICTS_PATH)

‎src/apify_client/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
ImpitHttpClient,
99
ImpitHttpClientAsync,
1010
)
11-
from ._types import Timeout
11+
from ._literals import Timeout
1212

1313
__version__ = metadata.version('apify-client')
1414

‎src/apify_client/_consts.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from typing import TYPE_CHECKING
55

66
if TYPE_CHECKING:
7-
from apify_client._models_generated import ActorJobStatus
7+
from apify_client._literals import TerminalActorJobStatus
88

99
DEFAULT_API_URL = 'https://api.apify.com'
1010
"""Default base URL for the Apify API."""
@@ -36,7 +36,7 @@
3636
DEFAULT_WAIT_WHEN_JOB_NOT_EXIST = timedelta(seconds=3)
3737
"""How long to wait for a job to exist before giving up."""
3838

39-
TERMINAL_STATUSES: frozenset[ActorJobStatus] = frozenset({'SUCCEEDED', 'FAILED', 'TIMED-OUT', 'ABORTED'})
39+
TERMINAL_STATUSES: frozenset[TerminalActorJobStatus] = frozenset({'SUCCEEDED', 'FAILED', 'TIMED-OUT', 'ABORTED'})
4040
"""Set of terminal Actor job statuses that indicate the job has finished."""
4141

4242
OVERRIDABLE_DEFAULT_HEADERS = {'Accept', 'Authorization', 'Accept-Encoding', 'User-Agent'}

‎src/apify_client/_http_clients/_base.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
if TYPE_CHECKING:
2626
from collections.abc import AsyncIterator, Iterator, Mapping
2727

28-
from apify_client._types import JsonSerializable, Timeout
28+
from apify_client._literals import JsonSerializable, Timeout
2929

3030

3131
@docs_group('HTTP clients')

‎src/apify_client/_http_clients/_impit.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
from collections.abc import Awaitable, Callable
2929

3030
from apify_client._http_clients import HttpResponse
31+
from apify_client._literals import JsonSerializable, Timeout
3132
from apify_client._statistics import ClientStatistics
32-
from apify_client._types import JsonSerializable, Timeout
3333

3434
T = TypeVar('T')
3535

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from __future__ import annotations
22

33
from datetime import timedelta
4-
from typing import Literal
4+
from typing import Literal, TypeAlias
55

66
from apify_client._models import WebhookRepresentation
77
from apify_client._models_generated import WebhookCreate
@@ -19,6 +19,9 @@
1919
`condition`) are ignored at runtime.
2020
"""
2121

22+
TerminalActorJobStatus: TypeAlias = Literal['SUCCEEDED', 'FAILED', 'TIMED-OUT', 'ABORTED']
23+
"""Subset of `ActorJobStatus` values that indicate the job has finished and will not change again."""
24+
2225
Timeout = timedelta | Literal['no_timeout', 'short', 'medium', 'long']
2326
"""Type for the `timeout` parameter on resource client methods.
2427

0 commit comments

Comments
 (0)