Skip to content

Commit ee0b011

Browse files
authored
refactor: derive Pydantic model field aliases with to_camel (#950)
Closes #949. Replaces hand-written per-field `Field(alias=...)` declarations with Pydantic's `alias_generator=to_camel`, so each model derives its camelCase JSON aliases from its snake_case field names. Following the existing convention in this repo, the config (`ConfigDict(populate_by_name=True, alias_generator=to_camel, ...)`) is repeated per model rather than introduced through a shared base class. ### Scope - `_charging.py` — the pricing-info and charge-event models. - `events/_types.py` — `SystemInfoEventData` and the event wrapper models. - `request_loaders/_apify_request_list.py` — the request-input models. - `storage_clients/_apify/_models.py` — `ApifyKeyValueStoreMetadata`, `RequestQueueHead`, `CachedRequest`, `ApifyRequestQueueMetadata`. ### No breaking change The generated aliases match the aliases that were declared by hand, so validation and serialization keep the same wire keys. Attribute names are unchanged (`run.usage.actor_compute_units` and friends still work). The one field whose name differs from its wire key is `RequestQueueHead.lock_time`. The API sends `lockSecs`, while `to_camel` would derive `lockTime`. It keeps `serialization_alias='lockSecs'` so it still emits `lockSecs`, and it now validates from both `lockSecs` and `lockTime` via `AliasChoices`. That change is additive. Nothing that parsed before stops parsing. `Configuration` is intentionally left untouched. Its fields bind to `APIFY_*` / `CRAWLEE_*` environment variables through validation aliases, so applying `to_camel` there would break env-based configuration.
1 parent cfff7bb commit ee0b011

4 files changed

Lines changed: 92 additions & 43 deletions

File tree

‎src/apify/_charging.py‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@
55
from dataclasses import dataclass
66
from datetime import UTC, datetime
77
from decimal import Decimal
8-
from typing import TYPE_CHECKING, Annotated, Literal, Protocol, TypedDict
8+
from typing import TYPE_CHECKING, Literal, Protocol, TypedDict
99

10-
from pydantic import Field
10+
from pydantic import ConfigDict
11+
from pydantic.alias_generators import to_camel
1112

1213
import apify_client._models as _client_models
1314
from apify_client._models import ActorChargeEvent as ClientActorChargeEvent
@@ -56,7 +57,9 @@
5657
class ActorChargeEvent(ClientActorChargeEvent):
5758
"""Definition of a single chargeable event in the pay-per-event pricing model."""
5859

59-
event_description: Annotated[str | None, Field(alias='eventDescription')] = None
60+
model_config = ConfigDict(alias_generator=to_camel)
61+
62+
event_description: str | None = None
6063
"""Human-readable description of the event.
6164
6265
Required in apify-client but omitted from the env var, so it is relaxed to optional.
@@ -67,75 +70,85 @@ class ActorChargeEvent(ClientActorChargeEvent):
6770
class PricingPerEvent(ClientPricingPerEvent):
6871
"""Pay-per-event pricing details - the chargeable events and their prices."""
6972

70-
actor_charge_events: Annotated[dict[str, ActorChargeEvent] | None, Field(alias='actorChargeEvents')] = None
73+
model_config = ConfigDict(alias_generator=to_camel)
74+
75+
actor_charge_events: dict[str, ActorChargeEvent] | None = None
7176
"""Mapping of event name to its charge definition."""
7277

7378

7479
@docs_group('Charging')
7580
class FreeActorPricingInfo(ClientFree):
7681
"""Pricing info for an Actor offered free of charge."""
7782

78-
apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
83+
model_config = ConfigDict(alias_generator=to_camel)
84+
85+
apify_margin_percentage: float | None = None
7986
"""Apify's margin on the price, as a percentage."""
8087

81-
created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
88+
created_at: datetime | None = None
8289
"""Timestamp when this pricing info was created."""
8390

84-
started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
91+
started_at: datetime | None = None
8592
"""Timestamp when this pricing became effective."""
8693

8794

8895
@docs_group('Charging')
8996
class FlatPricePerMonthActorPricingInfo(ClientFlatPricePerMonth):
9097
"""Pricing info for an Actor billed at a flat monthly price."""
9198

92-
apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
99+
model_config = ConfigDict(alias_generator=to_camel)
100+
101+
apify_margin_percentage: float | None = None
93102
"""Apify's margin on the price, as a percentage."""
94103

95-
created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
104+
created_at: datetime | None = None
96105
"""Timestamp when this pricing info was created."""
97106

98-
started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
107+
started_at: datetime | None = None
99108
"""Timestamp when this pricing became effective."""
100109

101-
trial_minutes: Annotated[int | None, Field(alias='trialMinutes')] = None
110+
trial_minutes: int | None = None
102111
"""Length of the free trial period, in minutes."""
103112

104-
price_per_unit_usd: Annotated[float | None, Field(alias='pricePerUnitUsd')] = None
113+
price_per_unit_usd: float | None = None
105114
"""Price per unit, in USD."""
106115

107116

108117
@docs_group('Charging')
109118
class PricePerDatasetItemActorPricingInfo(ClientPricePerDatasetItem):
110119
"""Pricing info for an Actor billed per dataset item produced."""
111120

112-
apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
121+
model_config = ConfigDict(alias_generator=to_camel)
122+
123+
apify_margin_percentage: float | None = None
113124
"""Apify's margin on the price, as a percentage."""
114125

115-
created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
126+
created_at: datetime | None = None
116127
"""Timestamp when this pricing info was created."""
117128

118-
started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
129+
started_at: datetime | None = None
119130
"""Timestamp when this pricing became effective."""
120131

121-
unit_name: Annotated[str | None, Field(alias='unitName')] = None
132+
unit_name: str | None = None
122133
"""Name of the billed unit."""
123134

124135

125136
@docs_group('Charging')
126137
class PayPerEventActorPricingInfo(ClientPayPerEvent):
127138
"""Pricing info for an Actor billed per charged event."""
128139

129-
apify_margin_percentage: Annotated[float | None, Field(alias='apifyMarginPercentage')] = None
140+
model_config = ConfigDict(alias_generator=to_camel)
141+
142+
apify_margin_percentage: float | None = None
130143
"""Apify's margin on the price, as a percentage."""
131144

132-
created_at: Annotated[datetime | None, Field(alias='createdAt')] = None
145+
created_at: datetime | None = None
133146
"""Timestamp when this pricing info was created."""
134147

135-
started_at: Annotated[datetime | None, Field(alias='startedAt')] = None
148+
started_at: datetime | None = None
136149
"""Timestamp when this pricing became effective."""
137150

138-
pricing_per_event: Annotated[PricingPerEvent, Field(alias='pricingPerEvent')]
151+
pricing_per_event: PricingPerEvent
139152
"""The pay-per-event pricing details."""
140153

141154

‎src/apify/events/_types.py‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
from datetime import datetime
44
from typing import Annotated, Any, Literal
55

6-
from pydantic import BaseModel, Field
6+
from pydantic import BaseModel, ConfigDict, Field
7+
from pydantic.alias_generators import to_camel
78

89
from crawlee.events._types import (
910
Event,
@@ -29,28 +30,30 @@
2930
class SystemInfoEventData(BaseModel):
3031
"""Resource usage metrics carried by a `systemInfo` event."""
3132

32-
mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
33+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
34+
35+
mem_avg_bytes: float
3336
"""Average memory usage over the measured interval, in bytes."""
3437

35-
mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
38+
mem_current_bytes: float
3639
"""Current memory usage, in bytes."""
3740

38-
mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
41+
mem_max_bytes: float
3942
"""Peak memory usage observed so far, in bytes."""
4043

41-
cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
44+
cpu_avg_usage: float
4245
"""Average CPU usage over the measured interval, in percent."""
4346

44-
cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
47+
cpu_max_usage: float
4548
"""Peak CPU usage observed so far, in percent."""
4649

47-
cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
50+
cpu_current_usage: float
4851
"""Current CPU usage, in percent."""
4952

50-
is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
53+
is_cpu_overloaded: bool
5154
"""Whether the CPU is currently overloaded."""
5255

53-
created_at: Annotated[datetime, Field(alias='createdAt')]
56+
created_at: datetime
5457
"""Timestamp when the metrics were collected."""
5558

5659
def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
@@ -73,6 +76,8 @@ def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
7376
class PersistStateEvent(BaseModel):
7477
"""A `persistState` event instructing the Actor to persist its state."""
7578

79+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
80+
7681
name: Literal[Event.PERSIST_STATE]
7782
"""The event name."""
7883

@@ -84,6 +89,8 @@ class PersistStateEvent(BaseModel):
8489
class SystemInfoEvent(BaseModel):
8590
"""A `systemInfo` event carrying the Actor's resource usage metrics."""
8691

92+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
93+
8794
name: Literal[Event.SYSTEM_INFO]
8895
"""The event name."""
8996

@@ -95,6 +102,8 @@ class SystemInfoEvent(BaseModel):
95102
class MigratingEvent(BaseModel):
96103
"""A `migrating` event signalling the Actor is about to be migrated to another host."""
97104

105+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
106+
98107
name: Literal[Event.MIGRATING]
99108
"""The event name."""
100109

@@ -106,6 +115,8 @@ class MigratingEvent(BaseModel):
106115
class AbortingEvent(BaseModel):
107116
"""An `aborting` event signalling the Actor run is being aborted."""
108117

118+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
119+
109120
name: Literal[Event.ABORTING]
110121
"""The event name."""
111122

@@ -117,6 +128,8 @@ class AbortingEvent(BaseModel):
117128
class ExitEvent(BaseModel):
118129
"""An `exit` event signalling the Actor process is about to exit."""
119130

131+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
132+
120133
name: Literal[Event.EXIT]
121134
"""The event name."""
122135

@@ -128,6 +141,8 @@ class ExitEvent(BaseModel):
128141
class EventWithoutData(BaseModel):
129142
"""A framework-level event that carries no payload (e.g. browser and page lifecycle events)."""
130143

144+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
145+
131146
name: Literal[
132147
Event.SESSION_RETIRED,
133148
Event.BROWSER_LAUNCHED,
@@ -146,6 +161,8 @@ class EventWithoutData(BaseModel):
146161
class DeprecatedEvent(BaseModel):
147162
"""A deprecated event kept for backward compatibility (e.g. `cpuInfo`)."""
148163

164+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
165+
149166
name: Literal['cpuInfo']
150167
"""The event name."""
151168

@@ -157,6 +174,8 @@ class DeprecatedEvent(BaseModel):
157174
class UnknownEvent(BaseModel):
158175
"""A fallback for any event whose name is not recognized by the SDK."""
159176

177+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
178+
160179
name: str
161180
"""The event name."""
162181

‎src/apify/request_loaders/_apify_request_list.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
from itertools import chain
66
from typing import Annotated, Any
77

8-
from pydantic import BaseModel, Field, TypeAdapter
8+
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
9+
from pydantic.alias_generators import to_camel
910

1011
from crawlee._types import HttpMethod
1112
from crawlee.http_clients import HttpClient, ImpitHttpClient
@@ -20,14 +21,16 @@
2021

2122

2223
class _RequestDetails(BaseModel):
24+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
25+
2326
method: HttpMethod = 'GET'
2427
payload: str = ''
2528
headers: Annotated[dict[str, str], Field(default_factory=dict)]
26-
user_data: Annotated[dict[str, str], Field(default_factory=dict, alias='userData')]
29+
user_data: Annotated[dict[str, str], Field(default_factory=dict)]
2730

2831

2932
class _RequestsFromUrlInput(_RequestDetails):
30-
requests_from_url: str = Field(alias='requestsFromUrl')
33+
requests_from_url: str
3134

3235

3336
class _SimpleUrlInput(_RequestDetails):

‎src/apify/storage_clients/_apify/_models.py‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
from datetime import datetime, timedelta
44
from typing import TYPE_CHECKING, Annotated
55

6-
from pydantic import BaseModel, ConfigDict, Field
6+
from pydantic import AliasChoices, BaseModel, ConfigDict, Field
7+
from pydantic.alias_generators import to_camel
78

89
from apify_client._models import RequestQueueStats
910
from crawlee.storage_clients.models import KeyValueStoreMetadata, RequestQueueMetadata
@@ -22,7 +23,9 @@ class ApifyKeyValueStoreMetadata(KeyValueStoreMetadata):
2223
Includes additional Apify-specific fields.
2324
"""
2425

25-
url_signing_secret_key: Annotated[str | None, Field(alias='urlSigningSecretKey', default=None)]
26+
model_config = ConfigDict(alias_generator=to_camel)
27+
28+
url_signing_secret_key: str | None = None
2629
"""The secret key used for signing URLs for secure access to key-value store records."""
2730

2831

@@ -34,24 +37,31 @@ class RequestQueueHead(BaseModel):
3437
including metadata about the queue's state and lock information for the requests.
3538
"""
3639

37-
model_config = ConfigDict(populate_by_name=True, extra='allow')
40+
model_config = ConfigDict(populate_by_name=True, extra='allow', alias_generator=to_camel)
3841

39-
limit: Annotated[int | None, Field(alias='limit', default=None)]
42+
limit: int | None = None
4043
"""The maximum number of requests that were requested from the queue."""
4144

42-
had_multiple_clients: Annotated[bool, Field(alias='hadMultipleClients', default=False)]
45+
had_multiple_clients: bool = False
4346
"""Indicates whether the queue has been accessed by multiple clients (consumers)."""
4447

45-
queue_modified_at: Annotated[datetime, Field(alias='queueModifiedAt')]
48+
queue_modified_at: datetime
4649
"""The timestamp when the queue was last modified."""
4750

48-
lock_time: Annotated[timedelta | None, Field(alias='lockSecs', default=None)]
49-
"""The duration for which the returned requests are locked and cannot be processed by other clients."""
51+
lock_time: Annotated[
52+
timedelta | None,
53+
Field(validation_alias=AliasChoices('lockSecs', 'lockTime'), serialization_alias='lockSecs'),
54+
] = None
55+
"""The duration for which the returned requests are locked and cannot be processed by other clients.
56+
57+
The platform's API names this field `lockSecs`, so it is serialized under that alias instead of the
58+
`lockTime` that `to_camel` would derive from the field name.
59+
"""
5060

51-
queue_has_locked_requests: Annotated[bool | None, Field(alias='queueHasLockedRequests', default=False)]
61+
queue_has_locked_requests: bool | None = False
5262
"""Indicates whether the queue contains any locked requests."""
5363

54-
items: Annotated[list[Request], Field(alias='items', default_factory=list[Request])]
64+
items: Annotated[list[Request], Field(default_factory=list[Request])]
5565
"""The list of request objects retrieved from the beginning of the queue."""
5666

5767
@classmethod
@@ -77,6 +87,8 @@ class CachedRequest(BaseModel):
7787
Only internal structure.
7888
"""
7989

90+
model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)
91+
8092
id: str
8193
"""Id of the request."""
8294

@@ -91,5 +103,7 @@ class CachedRequest(BaseModel):
91103

92104

93105
class ApifyRequestQueueMetadata(RequestQueueMetadata):
94-
stats: Annotated[RequestQueueStats, Field(alias='stats', default_factory=RequestQueueStats)]
106+
model_config = ConfigDict(alias_generator=to_camel)
107+
108+
stats: Annotated[RequestQueueStats, Field(default_factory=RequestQueueStats)]
95109
"""Additional statistics about the request queue."""

0 commit comments

Comments
 (0)