Repository navigation
Expand file tree
/
Copy pathtest_client_streaming.py
More file actions
129 lines (104 loc) · 5.04 KB
/
Copy pathtest_client_streaming.py
File metadata and controls
129 lines (104 loc) · 5.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
from __future__ import annotations
from typing import TYPE_CHECKING
from apify_client import ApifyClient, ApifyClientAsync
from apify_client.http_clients import HttpResponse
if TYPE_CHECKING:
from typing import Any
from pytest_httpserver import HTTPServer
from apify_client.http_clients import HttpClient, HttpClientAsync
DATASET_ID = 'test-dataset-id'
KVS_ID = 'test-kvs-id'
RECORD_KEY = 'test-record-key'
STREAM_CONTENT = b'[{"id": 1}]'
def test_dataset_stream_items_sync(
httpserver: HTTPServer,
http_client_class: type[HttpClient],
) -> None:
"""Dataset streams expose a transport-independent response that can be read synchronously."""
httpserver.expect_request(f'/v2/datasets/{DATASET_ID}/items').respond_with_data(STREAM_CONTENT)
api_url = httpserver.url_for('/').removesuffix('/')
client = ApifyClient.with_custom_http_client(
api_url=api_url,
http_client=http_client_class(),
)
with client.dataset(DATASET_ID).stream_items(item_format='json') as response:
assert isinstance(response, HttpResponse)
assert response.read() == STREAM_CONTENT
async def test_dataset_stream_items_async(
httpserver: HTTPServer,
http_client_async_class: type[HttpClientAsync],
) -> None:
"""Dataset streams expose a transport-independent response that can be read asynchronously."""
httpserver.expect_request(f'/v2/datasets/{DATASET_ID}/items').respond_with_data(STREAM_CONTENT)
api_url = httpserver.url_for('/').removesuffix('/')
client = ApifyClientAsync.with_custom_http_client(
api_url=api_url,
http_client=http_client_async_class(),
)
async with client.dataset(DATASET_ID).stream_items(item_format='json') as response:
assert isinstance(response, HttpResponse)
assert await response.aread() == STREAM_CONTENT
def test_key_value_store_stream_record_sync(
httpserver: HTTPServer,
http_client_class: type[HttpClient],
) -> None:
"""KVS streams require reading the generic response before consuming its content."""
httpserver.expect_request(f'/v2/key-value-stores/{KVS_ID}/records/{RECORD_KEY}').respond_with_data(STREAM_CONTENT)
api_url = httpserver.url_for('/').removesuffix('/')
client = ApifyClient.with_custom_http_client(
api_url=api_url,
http_client=http_client_class(),
)
with client.key_value_store(KVS_ID).stream_record(RECORD_KEY) as record:
assert isinstance(record, dict)
response = record['value']
assert isinstance(response, HttpResponse)
assert response.read() == STREAM_CONTENT
async def test_key_value_store_stream_record_async(
httpserver: HTTPServer,
http_client_async_class: type[HttpClientAsync],
) -> None:
"""KVS streams require asynchronously reading the generic response before consuming its content."""
httpserver.expect_request(f'/v2/key-value-stores/{KVS_ID}/records/{RECORD_KEY}').respond_with_data(STREAM_CONTENT)
api_url = httpserver.url_for('/').removesuffix('/')
client = ApifyClientAsync.with_custom_http_client(
api_url=api_url,
http_client=http_client_async_class(),
)
async with client.key_value_store(KVS_ID).stream_record(RECORD_KEY) as record:
assert isinstance(record, dict)
response = record['value']
assert isinstance(response, HttpResponse)
assert await response.aread() == STREAM_CONTENT
def test_protocol_check_leaves_stream_unread_sync(
httpserver: HTTPServer,
http_client_class: type[HttpClient],
) -> None:
"""Checking a streaming response against the protocol inspects it without pulling the body off the wire."""
httpserver.expect_request(f'/v2/datasets/{DATASET_ID}/items').respond_with_data(STREAM_CONTENT)
api_url = httpserver.url_for('/').removesuffix('/')
client = ApifyClient.with_custom_http_client(
api_url=api_url,
http_client=http_client_class(),
)
with client.dataset(DATASET_ID).stream_items(item_format='json') as response:
assert isinstance(response, HttpResponse)
# `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it.
raw: Any = response
assert raw.is_stream_consumed is False
async def test_protocol_check_leaves_stream_unread_async(
httpserver: HTTPServer,
http_client_async_class: type[HttpClientAsync],
) -> None:
"""Checking a streaming response against the protocol inspects it without pulling the body off the wire."""
httpserver.expect_request(f'/v2/datasets/{DATASET_ID}/items').respond_with_data(STREAM_CONTENT)
api_url = httpserver.url_for('/').removesuffix('/')
client = ApifyClientAsync.with_custom_http_client(
api_url=api_url,
http_client=http_client_async_class(),
)
async with client.dataset(DATASET_ID).stream_items(item_format='json') as response:
assert isinstance(response, HttpResponse)
# `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it.
raw: Any = response
assert raw.is_stream_consumed is False