Skip to content

Commit e76ef65

Browse files
committed
fix(files): keep the signed Host header when s3_endpoint_url rewrites the upload URL
With s3_endpoint_url set, FileClient.upload rewrote the presigned URL's host and sent the request with the new one. SigV4 binds the Host header, so the S3 server answered 403 SignatureDoesNotMatch; LocalStack's skipped signature checks had hidden it. The connection now goes to the override while the request carries the Host the URL was signed for.
1 parent 33c8fff commit e76ef65

4 files changed

Lines changed: 53 additions & 11 deletions

File tree

‎robosystems_client/clients/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ The package reads no environment variables on its own. `extract_token_from_envir
6868
| `max_retries` | `5` | Replays of a `429` response; reconnect attempts for `create_sse_client()` |
6969
| `retry_delay` | `1000` | Base backoff in milliseconds for both of the above |
7070
| `timeout` | `30` | Seconds, for ledger, investor and library GraphQL reads and `create_sse_client()`. Other REST calls set no client timeout |
71-
| `s3_endpoint_url` | `None` | Rewrites the host of file upload URLs, for example for LocalStack |
71+
| `s3_endpoint_url` | `None` | Where the client connects for file uploads when it reaches S3 by a different name than the API signed for (a container reaching a dev stack through `host.docker.internal`). The signed `Host` header is kept, as SigV4 requires |
7272
| `token_provider` | `None` | Callable returning the current credential |
7373

7474
## Cypher queries

‎robosystems_client/clients/facade.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ class RoboSystemsClientConfig:
2828
max_retries: int = 5
2929
retry_delay: int = 1000
3030
timeout: int = 30
31-
s3_endpoint_url: Optional[str] = None # Override S3 endpoint (e.g., for LocalStack)
31+
# Where the client reaches S3 when that is not the host the API signed for.
32+
s3_endpoint_url: Optional[str] = None
3233
# Zero-arg callable returning the current auth credential, consulted
3334
# on every request by the GraphQL-backed facades (ledger / investor /
3435
# library). Wins over the static token when set — use it when the

‎robosystems_client/clients/file_client.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ def upload(
109109
graph_id: Graph database identifier
110110
table_name: Table to associate file with
111111
file_or_buffer: File path, Path object, BytesIO, or file-like object
112-
options: Upload options (progress callback, LocalStack URL fix, auto-ingest)
112+
options: Upload options (progress callback, auto-ingest)
113113
114114
Returns:
115115
FileUploadResult with file metadata and status
@@ -184,13 +184,15 @@ def upload(
184184
upload_url = upload_data.get("upload_url")
185185
file_id = upload_data.get("file_id")
186186

187-
# Override S3 endpoint if configured (e.g., for LocalStack)
187+
# The client may reach S3 by a different name than the API signed for
188+
# (a container reaching a dev stack through host.docker.internal). SigV4
189+
# binds the Host header, so the request keeps the signed one.
190+
upload_headers = {"Content-Type": "application/x-parquet"}
188191
if self.s3_endpoint_url:
189192
from urllib.parse import urlparse, urlunparse
190193

191194
parsed_url = urlparse(upload_url)
192195
override_parsed = urlparse(self.s3_endpoint_url)
193-
# Replace scheme, host, and port with the override endpoint
194196
upload_url = urlunparse(
195197
(
196198
override_parsed.scheme or parsed_url.scheme,
@@ -201,6 +203,7 @@ def upload(
201203
parsed_url.fragment,
202204
)
203205
)
206+
upload_headers["Host"] = parsed_url.netloc
204207

205208
# Step 2: Upload file to S3
206209
if options.on_progress:
@@ -209,7 +212,7 @@ def upload(
209212
s3_response = self._http_client.put(
210213
upload_url,
211214
content=file_content,
212-
headers={"Content-Type": "application/x-parquet"},
215+
headers=upload_headers,
213216
)
214217

215218
if s3_response.status_code not in [200, 204]:

‎tests/test_file_client.py‎

Lines changed: 43 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -294,14 +294,15 @@ def test_upload_with_progress_callback(
294294
def test_upload_with_s3_endpoint_override(
295295
self, mock_create, mock_update, mock_config, graph_id
296296
):
297-
"""Test upload rewrites S3 URL when s3_endpoint_url is set."""
298-
mock_config["s3_endpoint_url"] = "http://localhost:4566"
297+
"""The connection goes to the override; the Host header stays the signed
298+
one, since SigV4 binds it and the server checks the signature against it."""
299+
mock_config["s3_endpoint_url"] = "http://host.docker.internal:8333"
299300

300301
mock_create_resp = Mock()
301302
mock_create_resp.status_code = 200
302303
mock_create_resp.parsed = Mock(
303304
result={
304-
"upload_url": "https://s3.amazonaws.com/bucket/file.parquet?sig=abc",
305+
"upload_url": "http://localhost:8333/bucket/file.parquet?X-Amz-Signature=abc",
305306
"file_id": "file-s3-override",
306307
}
307308
)
@@ -318,9 +319,46 @@ def test_upload_with_s3_endpoint_override(
318319

319320
client.upload(graph_id, "Entity", BytesIO(b"data"))
320321

321-
# Verify S3 PUT was called with overridden URL
322322
put_url = client._http_client.put.call_args[0][0]
323-
assert "localhost:4566" in put_url
323+
put_headers = client._http_client.put.call_args.kwargs["headers"]
324+
assert put_url == (
325+
"http://host.docker.internal:8333/bucket/file.parquet?X-Amz-Signature=abc"
326+
)
327+
assert put_headers["Host"] == "localhost:8333"
328+
assert put_headers["Content-Type"] == "application/x-parquet"
329+
330+
@patch("robosystems_client.clients.file_client.ingest_file")
331+
@patch("robosystems_client.clients.file_client.create_file_upload")
332+
def test_upload_without_override_sends_no_host_header(
333+
self, mock_create, mock_update, mock_config, graph_id
334+
):
335+
"""Without an override the presigned URL is used as signed and the
336+
client sets no Host header of its own."""
337+
mock_create_resp = Mock()
338+
mock_create_resp.status_code = 200
339+
mock_create_resp.parsed = Mock(
340+
result={
341+
"upload_url": "https://bucket.s3.amazonaws.com/file.parquet?X-Amz-Signature=abc",
342+
"file_id": "file-plain",
343+
}
344+
)
345+
mock_create.return_value = mock_create_resp
346+
347+
mock_update_resp = Mock()
348+
mock_update_resp.status_code = 200
349+
mock_update_resp.parsed = Mock(result={"file_size_bytes": 100, "row_count": 5})
350+
mock_update.return_value = mock_update_resp
351+
352+
client = FileClient(mock_config)
353+
client._http_client = Mock()
354+
client._http_client.put.return_value = Mock(status_code=200)
355+
356+
client.upload(graph_id, "Entity", BytesIO(b"data"))
357+
358+
put_url = client._http_client.put.call_args[0][0]
359+
put_headers = client._http_client.put.call_args.kwargs["headers"]
360+
assert put_url == "https://bucket.s3.amazonaws.com/file.parquet?X-Amz-Signature=abc"
361+
assert "Host" not in put_headers
324362

325363

326364
@pytest.mark.unit

0 commit comments

Comments
 (0)