Skip to content

Commit ecccadb

Browse files
authored
feat: add a user_agent_suffix option (#1145)
Adds `user_agent_suffix` to `ApifyClient`, `ApifyClientAsync` and the built-in HTTP clients, so tools built on top of the client can identify themselves. It takes a string or a list of strings and appends them to the default `User-Agent` after `; `, the same format as the JS client's `userAgentSuffix`. With `with_custom_http_client`, pass it to the HTTP client's constructor. Closes: #1148 *✍️ Drafted by Claude Code*
1 parent a31c749 commit ecccadb

6 files changed

Lines changed: 108 additions & 0 deletions

File tree

‎docs/02_concepts/10_custom_http_clients.mdx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ You can configure the default client through the <ApiLink to="class/ApifyClient"
4545
</TabItem>
4646
</Tabs>
4747

48+
To identify a tool built on top of the client, pass `user_agent_suffix`. The client appends it to its default `User-Agent` header after a semicolon and a space. Pass a list, such as `['MyTool/1.0', 'MyPlugin/2.0']`, to append several parts. The built-in HTTP clients accept the same option, so with `with_custom_http_client`, pass it to the HTTP client's constructor.
49+
4850
## Built-in HTTPX2 client
4951

5052
The package also provides <ApiLink to="class/Httpx2HttpClient">`Httpx2HttpClient`</ApiLink> and <ApiLink to="class/Httpx2HttpClientAsync">`Httpx2HttpClientAsync`</ApiLink>. They use the same request preparation, compression, retry policy, timeout tiers and growth, error handling, logging, and statistics as the default Impit clients, with [HTTPX2](https://github.com/pydantic/httpx2) as the transport.

‎src/apify_client/_apify_client.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ def __init__(
126126
timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
127127
timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
128128
headers: dict[str, str] | None = None,
129+
user_agent_suffix: str | list[str] | None = None,
129130
compression: HttpCompressionAlgorithm | HttpCompressor = 'gzip',
130131
) -> None:
131132
"""Initialize the Apify API client.
@@ -148,6 +149,9 @@ def __init__(
148149
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
149150
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
150151
headers: Additional HTTP headers to include in all API requests.
152+
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `, for
153+
example to identify the tool built on top of the client. With `with_custom_http_client`, pass it to
154+
the HTTP client's constructor.
151155
compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
152156
or an `HttpCompressor` instance for finer-grained control.
153157
@@ -218,6 +222,7 @@ def __init__(
218222
self._timeout_long = timeout_long
219223
self._timeout_max = timeout_max
220224
self._headers = headers
225+
self._user_agent_suffix = user_agent_suffix
221226
self._http_compressor = resolve_compressor(compression)
222227

223228
@classmethod
@@ -289,6 +294,7 @@ def http_client(self) -> HttpClient:
289294
min_delay_between_retries=self._min_delay_between_retries,
290295
statistics=self._statistics,
291296
headers=self._headers,
297+
user_agent_suffix=self._user_agent_suffix,
292298
http_compressor=self._http_compressor,
293299
)
294300

@@ -496,6 +502,7 @@ def __init__(
496502
timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
497503
timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
498504
headers: dict[str, str] | None = None,
505+
user_agent_suffix: str | list[str] | None = None,
499506
compression: HttpCompressionAlgorithm | HttpCompressor = 'gzip',
500507
) -> None:
501508
"""Initialize the Apify API client.
@@ -518,6 +525,9 @@ def __init__(
518525
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
519526
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
520527
headers: Additional HTTP headers to include in all API requests.
528+
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `, for
529+
example to identify the tool built on top of the client. With `with_custom_http_client`, pass it to
530+
the HTTP client's constructor.
521531
compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
522532
or an `HttpCompressor` instance for finer-grained control.
523533
@@ -588,6 +598,7 @@ def __init__(
588598
self._timeout_long = timeout_long
589599
self._timeout_max = timeout_max
590600
self._headers = headers
601+
self._user_agent_suffix = user_agent_suffix
591602
self._http_compressor = resolve_compressor(compression)
592603

593604
@classmethod
@@ -659,6 +670,7 @@ def http_client(self) -> HttpClientAsync:
659670
min_delay_between_retries=self._min_delay_between_retries,
660671
statistics=self._statistics,
661672
headers=self._headers,
673+
user_agent_suffix=self._user_agent_suffix,
662674
http_compressor=self._http_compressor,
663675
)
664676
return self._http_client

‎src/apify_client/http_clients/_base.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ def __init__(
126126
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
127127
statistics: ClientStatistics | None = None,
128128
headers: dict[str, str] | None = None,
129+
user_agent_suffix: str | list[str] | None = None,
129130
http_compressor: HttpCompressor | None = None,
130131
) -> None:
131132
"""Initialize the HTTP client base.
@@ -140,6 +141,7 @@ def __init__(
140141
min_delay_between_retries: Minimum delay between retries.
141142
statistics: Statistics tracker for API calls. Created automatically if not provided.
142143
headers: Additional HTTP headers to include in all requests.
144+
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
143145
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
144146
145147
Raises:
@@ -169,6 +171,9 @@ def __init__(
169171
client_version = metadata.version('apify-client')
170172

171173
user_agent = f'ApifyClient/{client_version} ({sys.platform}; Python/{python_version}); isAtHome/{is_at_home}'
174+
if user_agent_suffix:
175+
suffix_parts = [user_agent_suffix] if isinstance(user_agent_suffix, str) else user_agent_suffix
176+
user_agent += f'; {"; ".join(suffix_parts)}'
172177
default_headers['User-Agent'] = user_agent
173178

174179
if token is not None:

‎src/apify_client/http_clients/_httpx2.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ def __init__(
6565
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
6666
statistics: ClientStatistics | None = None,
6767
headers: dict[str, str] | None = None,
68+
user_agent_suffix: str | list[str] | None = None,
6869
http_compressor: HttpCompressor | None = None,
6970
) -> None:
7071
"""Initialize the HTTPX2-based synchronous HTTP client.
@@ -79,6 +80,7 @@ def __init__(
7980
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
8081
statistics: Statistics tracker for API calls. Created automatically if not provided.
8182
headers: Additional HTTP headers to include in all requests.
83+
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
8284
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
8385
"""
8486
super().__init__(
@@ -91,6 +93,7 @@ def __init__(
9193
min_delay_between_retries=min_delay_between_retries,
9294
statistics=statistics,
9395
headers=headers,
96+
user_agent_suffix=user_agent_suffix,
9497
http_compressor=http_compressor,
9598
)
9699

@@ -169,6 +172,7 @@ def __init__(
169172
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
170173
statistics: ClientStatistics | None = None,
171174
headers: dict[str, str] | None = None,
175+
user_agent_suffix: str | list[str] | None = None,
172176
http_compressor: HttpCompressor | None = None,
173177
) -> None:
174178
"""Initialize the HTTPX2-based asynchronous HTTP client.
@@ -183,6 +187,7 @@ def __init__(
183187
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
184188
statistics: Statistics tracker for API calls. Created automatically if not provided.
185189
headers: Additional HTTP headers to include in all requests.
190+
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
186191
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
187192
"""
188193
super().__init__(
@@ -195,6 +200,7 @@ def __init__(
195200
min_delay_between_retries=min_delay_between_retries,
196201
statistics=statistics,
197202
headers=headers,
203+
user_agent_suffix=user_agent_suffix,
198204
http_compressor=http_compressor,
199205
)
200206

‎src/apify_client/http_clients/_impit.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ def __init__(
6161
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
6262
statistics: ClientStatistics | None = None,
6363
headers: dict[str, str] | None = None,
64+
user_agent_suffix: str | list[str] | None = None,
6465
http_compressor: HttpCompressor | None = None,
6566
) -> None:
6667
"""Initialize the Impit-based synchronous HTTP client.
@@ -75,6 +76,7 @@ def __init__(
7576
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
7677
statistics: Statistics tracker for API calls. Created automatically if not provided.
7778
headers: Additional HTTP headers to include in all requests.
79+
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
7880
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
7981
"""
8082
super().__init__(
@@ -87,6 +89,7 @@ def __init__(
8789
min_delay_between_retries=min_delay_between_retries,
8890
statistics=statistics,
8991
headers=headers,
92+
user_agent_suffix=user_agent_suffix,
9093
http_compressor=http_compressor,
9194
)
9295

@@ -161,6 +164,7 @@ def __init__(
161164
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
162165
statistics: ClientStatistics | None = None,
163166
headers: dict[str, str] | None = None,
167+
user_agent_suffix: str | list[str] | None = None,
164168
http_compressor: HttpCompressor | None = None,
165169
) -> None:
166170
"""Initialize the Impit-based asynchronous HTTP client.
@@ -175,6 +179,7 @@ def __init__(
175179
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
176180
statistics: Statistics tracker for API calls. Created automatically if not provided.
177181
headers: Additional HTTP headers to include in all requests.
182+
user_agent_suffix: Extra part or parts appended to the default `User-Agent` header, separated by `; `.
178183
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
179184
"""
180185
super().__init__(
@@ -187,6 +192,7 @@ def __init__(
187192
min_delay_between_retries=min_delay_between_retries,
188193
statistics=statistics,
189194
headers=headers,
195+
user_agent_suffix=user_agent_suffix,
190196
http_compressor=http_compressor,
191197
)
192198

‎tests/unit/test_client_headers.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@
77
from typing import TYPE_CHECKING
88

99
import httpx2
10+
import pytest
1011
from werkzeug import Request, Response
1112

13+
from apify_client import ApifyClient, ApifyClientAsync
1214
from apify_client.http_clients import Httpx2HttpClient, Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync
1315

1416
if TYPE_CHECKING:
@@ -149,6 +151,81 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient
149151
assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings
150152

151153

154+
user_agent_suffix_cases = [
155+
pytest.param(None, '', id='no suffix'),
156+
pytest.param('MyTool/1.0', '; MyTool/1.0', id='string suffix'),
157+
pytest.param(['SDK/3.0.0', 'Crawlee/1.0.0'], '; SDK/3.0.0; Crawlee/1.0.0', id='list suffix'),
158+
]
159+
160+
161+
@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases)
162+
async def test_user_agent_suffix_async(
163+
httpserver: HTTPServer,
164+
http_client_async_class: type[HttpClientAsync],
165+
user_agent_suffix: str | list[str] | None,
166+
expected_suffix: str,
167+
) -> None:
168+
"""The HTTP client appends `user_agent_suffix` to the default `User-Agent` header."""
169+
client = http_client_async_class(user_agent_suffix=user_agent_suffix)
170+
httpserver.expect_request('/').respond_with_handler(_header_handler)
171+
172+
response = await client.call(method='GET', url=httpserver.url_for('/'))
173+
174+
request_headers = json.loads(response.text)['received_headers']
175+
assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}'
176+
177+
178+
@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases)
179+
def test_user_agent_suffix_sync(
180+
httpserver: HTTPServer,
181+
http_client_class: type[HttpClient],
182+
user_agent_suffix: str | list[str] | None,
183+
expected_suffix: str,
184+
) -> None:
185+
"""The HTTP client appends `user_agent_suffix` to the default `User-Agent` header."""
186+
client = http_client_class(user_agent_suffix=user_agent_suffix)
187+
httpserver.expect_request('/').respond_with_handler(_header_handler)
188+
189+
response = client.call(method='GET', url=httpserver.url_for('/'))
190+
191+
request_headers = json.loads(response.text)['received_headers']
192+
assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}'
193+
194+
195+
@pytest.mark.usefixtures('http_client_classes')
196+
@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases)
197+
async def test_apify_client_user_agent_suffix_async(
198+
httpserver: HTTPServer,
199+
user_agent_suffix: str | list[str] | None,
200+
expected_suffix: str,
201+
) -> None:
202+
"""`ApifyClientAsync` passes `user_agent_suffix` to its default HTTP client."""
203+
client = ApifyClientAsync(token='placeholder_token', user_agent_suffix=user_agent_suffix)
204+
httpserver.expect_request('/').respond_with_handler(_header_handler)
205+
206+
response = await client.http_client.call(method='GET', url=httpserver.url_for('/'))
207+
208+
request_headers = json.loads(response.text)['received_headers']
209+
assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}'
210+
211+
212+
@pytest.mark.usefixtures('http_client_classes')
213+
@pytest.mark.parametrize(('user_agent_suffix', 'expected_suffix'), user_agent_suffix_cases)
214+
def test_apify_client_user_agent_suffix_sync(
215+
httpserver: HTTPServer,
216+
user_agent_suffix: str | list[str] | None,
217+
expected_suffix: str,
218+
) -> None:
219+
"""`ApifyClient` passes `user_agent_suffix` to its default HTTP client."""
220+
client = ApifyClient(token='placeholder_token', user_agent_suffix=user_agent_suffix)
221+
httpserver.expect_request('/').respond_with_handler(_header_handler)
222+
223+
response = client.http_client.call(method='GET', url=httpserver.url_for('/'))
224+
225+
request_headers = json.loads(response.text)['received_headers']
226+
assert request_headers['User-Agent'] == f'{_get_user_agent()}{expected_suffix}'
227+
228+
152229
async def test_per_request_headers_override_defaults_async(
153230
httpserver: HTTPServer, http_client_async_class: type[HttpClientAsync]
154231
) -> None:

0 commit comments

Comments
 (0)