Skip to content

Commit 0889b88

Browse files
committed
Introduce fully typed clients
1 parent 1939b7f commit 0889b88

82 files changed

Lines changed: 6231 additions & 1751 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov integration-tests \
2+
integration-tests-cov format check-async-docstrings check-code fix-async-docstrings build-api-reference \
3+
build-docs run-docs
4+
5+
# This is default for local testing, but GitHub workflows override it to a higher value in CI
6+
INTEGRATION_TESTS_CONCURRENCY = 1
7+
8+
clean:
9+
rm -rf .ty_cache .pytest_cache .ruff_cache build dist htmlcov .coverage
10+
11+
install-dev:
12+
uv sync --all-extras
13+
uv run pre-commit install
14+
15+
build:
16+
uv build --verbose
17+
18+
# APIFY_PYPI_TOKEN_CRAWLEE is expected to be set in the environment
19+
publish-to-pypi:
20+
uv publish --verbose --token "${APIFY_PYPI_TOKEN_CRAWLEE}"
21+
22+
lint:
23+
uv run ruff format --check
24+
uv run ruff check
25+
26+
type-check:
27+
uv run ty check
28+
29+
unit-tests:
30+
uv run pytest \
31+
--numprocesses=auto \
32+
--verbose \
33+
tests/unit
34+
35+
unit-tests-cov:
36+
uv run pytest \
37+
--numprocesses=auto \
38+
--verbose \
39+
--cov=src/apify_client \
40+
--cov-report=xml:coverage-unit.xml \
41+
tests/unit
42+
43+
integration-tests:
44+
uv run pytest \
45+
--numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) \
46+
--verbose \
47+
tests/integration
48+
49+
integration-tests-cov:
50+
uv run pytest \
51+
--numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) \
52+
--verbose \
53+
--cov=src/apify_client \
54+
--cov-report=xml:coverage-integration.xml \
55+
tests/integration
56+
57+
format:
58+
uv run ruff check --fix
59+
uv run ruff format
60+
61+
check-async-docstrings:
62+
uv run python scripts/check_async_docstrings.py
63+
64+
# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks
65+
# and the run_checks.yaml GitHub Actions workflow.
66+
check-code: lint type-check unit-tests check-async-docstrings
67+
68+
generate-models:
69+
uv run datamodel-codegen
70+
71+
fix-async-docstrings:
72+
uv run python scripts/fix_async_docstrings.py
73+
74+
build-api-reference:
75+
cd website && uv run ./build_api_reference.sh
76+
77+
build-docs:
78+
cd website && uv run npm clean-install && uv run npm run build
79+
80+
run-docs: build-api-reference
81+
cd website && uv run npm clean-install && uv run npm run start

‎docs/01_overview/code/01_usage_async.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ async def main() -> None:
1616
return
1717

1818
# Fetch results from the Actor run's default dataset.
19-
dataset_client = apify_client.dataset(call_result['defaultDatasetId'])
19+
dataset_client = apify_client.dataset(call_result.default_dataset_id)
2020
list_items_result = await dataset_client.list_items()
2121
print(f'Dataset: {list_items_result}')

‎docs/01_overview/code/01_usage_sync.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ def main() -> None:
1616
return
1717

1818
# Fetch results from the Actor run's default dataset.
19-
dataset_client = apify_client.dataset(call_result['defaultDatasetId'])
19+
dataset_client = apify_client.dataset(call_result.default_dataset_id)
2020
list_items_result = dataset_client.list_items()
2121
print(f'Dataset: {list_items_result}')

‎docs/02_concepts/code/01_async_support.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ async def main() -> None:
1111

1212
# Start the Actor and get the run ID
1313
run_result = await actor_client.start()
14-
run_client = apify_client.run(run_result['id'])
14+
run_client = apify_client.run(run_result.id)
1515
log_client = run_client.log()
1616

1717
# Stream the logs

‎docs/03_examples/code/02_tasks_async.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,22 @@
11
import asyncio
22

33
from apify_client import ApifyClientAsync
4-
from apify_client.clients.resource_clients import TaskClientAsync
4+
from apify_client._models import Run, Task
5+
from apify_client._resource_clients import TaskClientAsync
56

67
TOKEN = 'MY-APIFY-TOKEN'
78
HASHTAGS = ['zebra', 'lion', 'hippo']
89

910

10-
async def run_apify_task(client: TaskClientAsync) -> dict:
11-
result = await client.call()
12-
return result or {}
11+
async def run_apify_task(client: TaskClientAsync) -> Run | None:
12+
return await client.call()
1313

1414

1515
async def main() -> None:
1616
apify_client = ApifyClientAsync(token=TOKEN)
1717

1818
# Create Apify tasks
19-
apify_tasks = list[dict]()
19+
apify_tasks = list[Task]()
2020
apify_tasks_client = apify_client.tasks()
2121

2222
for hashtag in HASHTAGS:
@@ -34,7 +34,7 @@ async def main() -> None:
3434
apify_task_clients = list[TaskClientAsync]()
3535

3636
for apify_task in apify_tasks:
37-
task_id = apify_task['id']
37+
task_id = apify_task.id
3838
apify_task_client = apify_client.task(task_id)
3939
apify_task_clients.append(apify_task_client)
4040

‎docs/03_examples/code/02_tasks_sync.py‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
from apify_client import ApifyClient
2-
from apify_client.clients.resource_clients import TaskClient
2+
from apify_client._models import Run, Task
3+
from apify_client._resource_clients import TaskClient
34

45
TOKEN = 'MY-APIFY-TOKEN'
56
HASHTAGS = ['zebra', 'lion', 'hippo']
67

78

8-
def run_apify_task(client: TaskClient) -> dict:
9-
result = client.call()
10-
return result or {}
9+
def run_apify_task(client: TaskClient) -> Run | None:
10+
return client.call()
1111

1212

1313
def main() -> None:
1414
apify_client = ApifyClient(token=TOKEN)
1515

1616
# Create Apify tasks
17-
apify_tasks = list[dict]()
17+
apify_tasks = list[Task]()
1818
apify_tasks_client = apify_client.tasks()
1919

2020
for hashtag in HASHTAGS:
@@ -32,18 +32,19 @@ def main() -> None:
3232
apify_task_clients = list[TaskClient]()
3333

3434
for apify_task in apify_tasks:
35-
task_id = apify_task['id']
35+
task_id = apify_task.id
3636
apify_task_client = apify_client.task(task_id)
3737
apify_task_clients.append(apify_task_client)
3838

3939
print('Task clients created:', apify_task_clients)
4040

4141
# Execute Apify tasks
42-
task_run_results = list[dict]()
42+
task_run_results = list[Run]()
4343

4444
for client in apify_task_clients:
4545
result = run_apify_task(client)
46-
task_run_results.append(result)
46+
if result is not None:
47+
task_run_results.append(result)
4748

4849
print('Task results:', task_run_results)
4950

‎docs/03_examples/code/03_retrieve_async.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@ async def main() -> None:
1919

2020
for dataset_item in actor_datasets.items:
2121
# Dataset items can be handled here. Dataset items can be paginated
22-
dataset_client = apify_client.dataset(dataset_item['id'])
22+
dataset_client = apify_client.dataset(dataset_item.id)
2323
dataset_items = await dataset_client.list_items(limit=1000)
2424

2525
# Items can be pushed to single dataset
26-
merging_dataset_client = apify_client.dataset(merging_dataset['id'])
26+
merging_dataset_client = apify_client.dataset(merging_dataset.id)
2727
await merging_dataset_client.push_items(dataset_items.items)
2828

2929
# ...

‎docs/03_examples/code/03_retrieve_sync.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@ def main() -> None:
1717

1818
for dataset_item in actor_datasets.items:
1919
# Dataset items can be handled here. Dataset items can be paginated
20-
dataset_client = apify_client.dataset(dataset_item['id'])
20+
dataset_client = apify_client.dataset(dataset_item.id)
2121
dataset_items = dataset_client.list_items(limit=1000)
2222

2323
# Items can be pushed to single dataset
24-
merging_dataset_client = apify_client.dataset(merging_dataset['id'])
24+
merging_dataset_client = apify_client.dataset(merging_dataset.id)
2525
merging_dataset_client.push_items(dataset_items.items)
2626

2727
# ...

‎pyproject.toml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ dev = [
5959
"ty~=0.0.0",
6060
"types-colorama<0.5.0",
6161
"werkzeug<4.0.0", # Werkzeug is used by pytest-httpserver
62+
"datamodel-code-generator[http,ruff]<1.0.0",
6263
]
6364

6465
[tool.hatch.build.targets.wheel]
@@ -139,6 +140,11 @@ indent-style = "space"
139140
"N999", # Invalid module name
140141
"T201", # print found
141142
]
143+
"src/apify_client/_models.py" = [
144+
"D", # Everything from the pydocstyle
145+
"E501", # Line too long
146+
"ERA001", # Commented-out code
147+
]
142148

143149
[tool.ruff.lint.flake8-quotes]
144150
docstring-quotes = "double"
@@ -180,6 +186,7 @@ exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"]
180186
[tool.ipdb]
181187
context = 7
182188

189+
<<<<<<< HEAD
183190
# Run tasks with: uv run poe <task>
184191
[tool.poe.tasks]
185192
clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov"
@@ -215,3 +222,23 @@ cwd = "website"
215222
[tool.poe.tasks.run-docs]
216223
shell = "./build_api_reference.sh && corepack enable && yarn && uv run yarn start"
217224
cwd = "website"
225+
=======
226+
# https://koxudaxi.github.io/datamodel-code-generator/
227+
[tool.datamodel-codegen]
228+
# url = "https://docs.apify.com/api/openapi.json"
229+
input = "../apify-docs/static/api/openapi.json"
230+
input_file_type = "openapi"
231+
output = "src/apify_client/_models.py"
232+
target_python_version = "3.10"
233+
output_model_type = "pydantic_v2.BaseModel"
234+
use_schema_description = true
235+
use_field_description = true
236+
use_union_operator = true
237+
capitalise_enum_members = true
238+
collapse_root_models = true
239+
set_default_enum_member = true
240+
use_annotated = true
241+
wrap_string_literal = true
242+
snake_case_field = true
243+
formatters = ["ruff-check", "ruff-format"]
244+
>>>>>>> af344d1 (Introduce fully typed clients)

‎scripts/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def get_current_package_version() -> str:
2525
# It replaces the version number on the line with the format `version = "1.2.3"`
2626
def set_current_package_version(version: str) -> None:
2727
with open(PYPROJECT_TOML_FILE_PATH, 'r+', encoding='utf-8') as pyproject_toml_file:
28-
updated_pyproject_toml_file_lines = []
28+
updated_pyproject_toml_file_lines = list[str]()
2929
version_string_found = False
3030
for line in pyproject_toml_file:
3131
line_processed = line

0 commit comments

Comments
 (0)