Skip to content

Commit 7e8623f

Browse files
Rusty Conoverclaude
andcommitted
fix(protocol): encode catalog items with map keys in sorted order
Arrow writes a map column in dict iteration order, so equal items whose tags / estimated_object_count / column_comments were filled in different orders encoded differently. catalog_contents promises deterministic, byte-identical items (and its content-hash etag hashes them); the Go, Java, C# and TypeScript SDKs already sort map keys. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent f3ccd7f commit 7e8623f

2 files changed

Lines changed: 53 additions & 2 deletions

File tree

‎tests/catalog/test_catalog_contents.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,3 +512,26 @@ def test_client_round_trip(client_transport: Any) -> None:
512512
assert hit.schemas == []
513513
assert hit.etag == "gen-1"
514514
client.catalog_detach(attach_opaque_data=reval.attach_opaque_data)
515+
516+
517+
def test_item_encoding_sorts_map_keys() -> None:
518+
"""Equal items whose maps were filled in different orders encode identically."""
519+
from vgi.catalog import SchemaInfo
520+
from vgi.protocol import _item_ipc_bytes
521+
522+
a = SchemaInfo(
523+
attach_opaque_data=b"x",
524+
path=["s"],
525+
comment=None,
526+
tags={"b": "2", "a": "1"},
527+
estimated_object_count={"view": 0, "table": 3},
528+
)
529+
b = SchemaInfo(
530+
attach_opaque_data=b"x",
531+
path=["s"],
532+
comment=None,
533+
tags={"a": "1", "b": "2"},
534+
estimated_object_count={"table": 3, "view": 0},
535+
)
536+
assert _item_ipc_bytes(a) == _item_ipc_bytes(b)
537+
assert SchemaInfo.deserialize_from_bytes(_item_ipc_bytes(a)).tags == {"a": "1", "b": "2"}

‎vgi/protocol.py‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -867,13 +867,41 @@ def _item_ipc_bytes(info: Any) -> bytes:
867867
"""
868868
memo = getattr(info, "__dict__", None)
869869
if memo is None:
870-
return info.serialize_to_bytes() # type: ignore[no-any-return]
870+
return _with_sorted_maps(info).serialize_to_bytes() # type: ignore[no-any-return]
871871
encoded = memo.get(_ITEM_IPC_MEMO)
872872
if encoded is None:
873-
encoded = memo[_ITEM_IPC_MEMO] = info.serialize_to_bytes()
873+
encoded = memo[_ITEM_IPC_MEMO] = _with_sorted_maps(info).serialize_to_bytes()
874874
return encoded # type: ignore[no-any-return]
875875

876876

877+
def _with_sorted_maps(info: Any) -> Any:
878+
"""Return *info* with every map-valued field in sorted key order.
879+
880+
Arrow writes a map column in the dict's iteration order, so two equal
881+
items whose ``tags`` (or ``estimated_object_count``, ``column_comments``,
882+
...) were filled in different orders would encode to different bytes.
883+
Catalog items must encode deterministically: ``catalog_contents`` promises
884+
items byte-identical to the per-schema RPCs, its content-hash etag hashes
885+
them, and every other VGI SDK sorts map keys the same way.
886+
887+
Args:
888+
info: The catalog item (a frozen dataclass).
889+
890+
Returns:
891+
*info* itself when no map field needs reordering, else a copy.
892+
"""
893+
if not dataclasses.is_dataclass(info) or isinstance(info, type):
894+
return info
895+
changes: dict[str, dict[Any, Any]] = {}
896+
for f in dataclasses.fields(info):
897+
if not f.init:
898+
continue
899+
value = getattr(info, f.name, None)
900+
if isinstance(value, dict) and list(value) != sorted(value):
901+
changes[f.name] = dict(sorted(value.items()))
902+
return dataclasses.replace(info, **changes) if changes else info
903+
904+
877905
def _catalog_items_response(item_type: type) -> type:
878906
"""Generate a catalog items response class for the given `ArrowSerializableDataclass` type.
879907

0 commit comments

Comments
 (0)