|
| 1 | +# Copyright 2025, 2026 Query Farm LLC - https://query.farm |
| 2 | + |
| 3 | +"""Parity between hand-built wire records and their declared ``ARROW_SCHEMA``. |
| 4 | +
|
| 5 | +Most wire records in this package derive their schema from the dataclass |
| 6 | +(``ArrowSerializableDataclass``), so the batch is built *from* ``ARROW_SCHEMA`` |
| 7 | +and drift is structurally impossible. A handful do not: they declare |
| 8 | +``ARROW_SCHEMA`` by hand *and* build the row by hand, and ``from_pylist`` is |
| 9 | +forgiving in both directions — a key the schema does not declare is dropped |
| 10 | +without a word, and a column the row omits is written as null. Neither shows up |
| 11 | +in the bytes, and to a reader both look like a worker that had nothing to say. |
| 12 | +
|
| 13 | +That failure mode is not hypothetical; it has cost the sibling SDKs real |
| 14 | +outages: |
| 15 | +
|
| 16 | +* TypeScript's ``ScanBranch`` builder supplied 7 of the schema's 9 columns |
| 17 | + because it was never updated when the two ``format_*`` fields were added. The |
| 18 | + missing ``format_locations`` is a LIST, and Arrow dereferences a list's |
| 19 | + children while writing, so EVERY multi-branch table in that worker died with |
| 20 | + ``TypeError: Cannot read properties of undefined (reading 'slice')`` — |
| 21 | + including tables that predated those fields entirely. |
| 22 | +* Java's ``PlanResponse`` marked two fields nullable where the protocol says |
| 23 | + non-null, and the client rejected the whole response as an "out-of-date |
| 24 | + Apache Arrow schema". |
| 25 | +
|
| 26 | +So for every such record this module builds one with every field populated and |
| 27 | +checks three things: the row's key set is exactly the schema's column set, the |
| 28 | +serialized batch's schema is exactly ``ARROW_SCHEMA``, and no column came back |
| 29 | +null. ``test_every_hand_declared_schema_is_covered`` then walks the package for |
| 30 | +hand-declared ``ARROW_SCHEMA`` class attributes, so a record type added later is |
| 31 | +either covered here or explicitly excused — never silently untested. |
| 32 | +""" |
| 33 | + |
| 34 | +from __future__ import annotations |
| 35 | + |
| 36 | +import importlib |
| 37 | +import pkgutil |
| 38 | +from collections.abc import Callable |
| 39 | +from dataclasses import dataclass |
| 40 | +from typing import Any |
| 41 | + |
| 42 | +import pyarrow as pa |
| 43 | +import pytest |
| 44 | +from vgi_rpc.utils import deserialize_record_batch |
| 45 | + |
| 46 | +import vgi |
| 47 | +from vgi.catalog.attach_option import AttachOptionSpec |
| 48 | +from vgi.catalog.catalog_interface import ( |
| 49 | + ScanBranch, |
| 50 | + ScanBranchesResult, |
| 51 | + ScanFunctionResult, |
| 52 | +) |
| 53 | +from vgi.catalog.secret_type import SecretTypeSpec |
| 54 | +from vgi.catalog.setting import SettingSpec |
| 55 | + |
| 56 | + |
| 57 | +@dataclass(frozen=True) |
| 58 | +class WireRecordCase: |
| 59 | + """One record that declares its own ``ARROW_SCHEMA`` and builds its own row. |
| 60 | +
|
| 61 | + ``build`` returns an instance with EVERY field of the schema populated, so a |
| 62 | + null column in the result means the builder dropped the value rather than |
| 63 | + the fixture never supplying one. |
| 64 | + """ |
| 65 | + |
| 66 | + name: str |
| 67 | + build: Callable[[], Any] |
| 68 | + |
| 69 | + |
| 70 | +def _scalar_args() -> tuple[list[pa.Scalar], dict[str, pa.Scalar]]: |
| 71 | + return ( |
| 72 | + [pa.scalar("s3://bucket/x.parquet", pa.string())], |
| 73 | + {"hive_partitioning": pa.scalar(True, pa.bool_())}, |
| 74 | + ) |
| 75 | + |
| 76 | + |
| 77 | +def _scan_function_result() -> ScanFunctionResult: |
| 78 | + positional, named = _scalar_args() |
| 79 | + return ScanFunctionResult( |
| 80 | + function_name="read_parquet", |
| 81 | + positional_arguments=positional, |
| 82 | + named_arguments=named, |
| 83 | + required_extensions=["parquet"], |
| 84 | + ) |
| 85 | + |
| 86 | + |
| 87 | +def _scan_branch() -> ScanBranch: |
| 88 | + positional, named = _scalar_args() |
| 89 | + return ScanBranch( |
| 90 | + function_name="read_parquet", |
| 91 | + positional_arguments=positional, |
| 92 | + named_arguments=named, |
| 93 | + branch_filter="ts >= '2026-01-01'", |
| 94 | + writable=True, |
| 95 | + source_catalog="lake", |
| 96 | + source_schema="main", |
| 97 | + source_table="events", |
| 98 | + format_name="parquet", |
| 99 | + format_locations=["s3://bucket/a.parquet"], |
| 100 | + ) |
| 101 | + |
| 102 | + |
| 103 | +def _scan_branches_result() -> ScanBranchesResult: |
| 104 | + return ScanBranchesResult(branches=[_scan_branch()], required_extensions=["parquet"]) |
| 105 | + |
| 106 | + |
| 107 | +CASES: list[WireRecordCase] = [ |
| 108 | + WireRecordCase("ScanFunctionResult", _scan_function_result), |
| 109 | + WireRecordCase("ScanBranch", _scan_branch), |
| 110 | + WireRecordCase("ScanBranchesResult", _scan_branches_result), |
| 111 | + WireRecordCase( |
| 112 | + "SettingSpec", |
| 113 | + lambda: SettingSpec(name="greeting", desc="Greeting text", type=pa.string(), default="hello"), |
| 114 | + ), |
| 115 | + WireRecordCase( |
| 116 | + # `required` and `default` are mutually exclusive by construction, so |
| 117 | + # this instance populates `default` and the `required` column carries |
| 118 | + # its meaningful False. The nullable-column check below skips it. |
| 119 | + "AttachOptionSpec", |
| 120 | + lambda: AttachOptionSpec(name="multiplier", desc="Row multiplier", type=pa.int64(), default=2, required=False), |
| 121 | + ), |
| 122 | + WireRecordCase( |
| 123 | + "SecretTypeSpec", |
| 124 | + lambda: SecretTypeSpec( |
| 125 | + name="vgi_example", |
| 126 | + description="Example VGI secret", |
| 127 | + schema=pa.schema([pa.field("api_key", pa.string(), metadata={"redact": "true"})]), |
| 128 | + ), |
| 129 | + ), |
| 130 | +] |
| 131 | + |
| 132 | +#: Hand-declared ``ARROW_SCHEMA`` holders that no case builds, and why. Listing |
| 133 | +#: them beats ignoring unknown classes: adding a record forces a deliberate |
| 134 | +#: choice between covering it and writing down why it needs no coverage. |
| 135 | +NOT_COVERED: dict[str, str] = { |
| 136 | + "_SpecBase": ( |
| 137 | + "abstract base — SettingSpec and AttachOptionSpec are the concrete records on the wire and both are covered" |
| 138 | + ), |
| 139 | +} |
| 140 | + |
| 141 | +CASES_BY_NAME = {case.name: case for case in CASES} |
| 142 | + |
| 143 | + |
| 144 | +@pytest.mark.parametrize("case", CASES, ids=lambda c: c.name) |
| 145 | +def test_row_dict_covers_every_declared_column(case: WireRecordCase) -> None: |
| 146 | + """The builder's row names exactly the columns the schema declares. |
| 147 | +
|
| 148 | + A missing key is the ScanBranch bug: silently null on the wire (or, for a |
| 149 | + list column, a crash inside Arrow's writer). An extra key is the opposite |
| 150 | + failure and just as quiet — ``from_pylist`` drops it, so a value the caller |
| 151 | + set never reaches the peer at all. |
| 152 | + """ |
| 153 | + record = case.build() |
| 154 | + schema: pa.Schema = type(record).ARROW_SCHEMA |
| 155 | + row_keys = set(record.to_row_dict()) |
| 156 | + declared = set(schema.names) |
| 157 | + |
| 158 | + assert row_keys == declared, ( |
| 159 | + f"{case.name}.to_row_dict() does not match ARROW_SCHEMA: " |
| 160 | + f"missing {sorted(declared - row_keys)}, unexpected {sorted(row_keys - declared)}" |
| 161 | + ) |
| 162 | + |
| 163 | + |
| 164 | +@pytest.mark.parametrize("case", CASES, ids=lambda c: c.name) |
| 165 | +def test_serialized_schema_is_the_declared_schema(case: WireRecordCase) -> None: |
| 166 | + """The bytes on the wire carry ``ARROW_SCHEMA``, field for field. |
| 167 | +
|
| 168 | + Names, order, types and nullability all matter: the C++ client matches the |
| 169 | + response schema against its own generated copy and rejects the whole |
| 170 | + response on any difference, which is how Java's two over-nullable fields |
| 171 | + turned into "out-of-date Apache Arrow schema". |
| 172 | + """ |
| 173 | + record = case.build() |
| 174 | + schema: pa.Schema = type(record).ARROW_SCHEMA |
| 175 | + batch, _ = deserialize_record_batch(record.serialize()) |
| 176 | + |
| 177 | + assert batch.schema.names == schema.names, ( |
| 178 | + f"{case.name}: serialized columns {batch.schema.names} != declared {schema.names}" |
| 179 | + ) |
| 180 | + for declared_field in schema: |
| 181 | + actual = batch.schema.field(declared_field.name) |
| 182 | + assert actual.type == declared_field.type, ( |
| 183 | + f"{case.name}.{declared_field.name}: serialized as {actual.type}, declared {declared_field.type}" |
| 184 | + ) |
| 185 | + assert actual.nullable == declared_field.nullable, ( |
| 186 | + f"{case.name}.{declared_field.name}: serialized nullable={actual.nullable}, " |
| 187 | + f"declared nullable={declared_field.nullable}" |
| 188 | + ) |
| 189 | + |
| 190 | + |
| 191 | +@pytest.mark.parametrize("case", CASES, ids=lambda c: c.name) |
| 192 | +def test_every_column_is_populated(case: WireRecordCase) -> None: |
| 193 | + """No column comes back null for a record whose every field is set. |
| 194 | +
|
| 195 | + A column that is present but never written is indistinguishable on the wire |
| 196 | + from "the worker has nothing to say", so the peer reads a default and the |
| 197 | + feature quietly does not exist. |
| 198 | + """ |
| 199 | + record = case.build() |
| 200 | + schema: pa.Schema = type(record).ARROW_SCHEMA |
| 201 | + batch, _ = deserialize_record_batch(record.serialize()) |
| 202 | + |
| 203 | + row = batch.to_pylist()[0] |
| 204 | + for name in schema.names: |
| 205 | + assert row[name] is not None, ( |
| 206 | + f"{case.name}.{name} is null even though the fixture populates every field " |
| 207 | + f"— the builder is not wiring this column" |
| 208 | + ) |
| 209 | + |
| 210 | + |
| 211 | +def _hand_declared_schema_classes() -> dict[str, type]: |
| 212 | + """Classes in the ``vgi`` package that declare ``ARROW_SCHEMA`` themselves. |
| 213 | +
|
| 214 | + Deliberately reads ``__dict__`` rather than ``getattr``: an |
| 215 | + ``ArrowSerializableDataclass`` subclass answers ``ARROW_SCHEMA`` from a |
| 216 | + descriptor that GENERATES it from the dataclass fields, so its batch cannot |
| 217 | + disagree with its schema. Only a class that spells the schema out by hand |
| 218 | + has two things that can drift apart, and only those belong here. |
| 219 | + """ |
| 220 | + found: dict[str, type] = {} |
| 221 | + for module_info in pkgutil.walk_packages(vgi.__path__, prefix="vgi."): |
| 222 | + try: |
| 223 | + module = importlib.import_module(module_info.name) |
| 224 | + except ImportError: # pragma: no cover - optional extras |
| 225 | + continue |
| 226 | + for obj in vars(module).values(): |
| 227 | + if not isinstance(obj, type) or obj.__module__ != module_info.name: |
| 228 | + continue |
| 229 | + if isinstance(vars(obj).get("ARROW_SCHEMA"), pa.Schema): |
| 230 | + # Keyed by __qualname__, not the binding name, so a module-level |
| 231 | + # alias (WriteFunctionResult = ScanFunctionResult) does not read |
| 232 | + # as a second, uncovered record type. |
| 233 | + found[obj.__qualname__] = obj |
| 234 | + return found |
| 235 | + |
| 236 | + |
| 237 | +def test_every_hand_declared_schema_is_covered() -> None: |
| 238 | + """No hand-built record type escapes the parity checks above. |
| 239 | +
|
| 240 | + A new record type FAILS here as unlisted rather than being picked up |
| 241 | + automatically. Auto-coverage would need a generic fully-populated builder |
| 242 | + per record, and there isn't one — the builders take dataclass fields, not |
| 243 | + schemas — so "automatic" would in practice mean "skipped", which recreates |
| 244 | + the gap this test exists to close. |
| 245 | + """ |
| 246 | + classes = _hand_declared_schema_classes() |
| 247 | + assert classes, ( |
| 248 | + "found no classes declaring their own ARROW_SCHEMA — the discovery walk broke " |
| 249 | + "and this coverage guard is now silently vacuous" |
| 250 | + ) |
| 251 | + |
| 252 | + missing = sorted(set(classes) - set(CASES_BY_NAME) - set(NOT_COVERED)) |
| 253 | + assert not missing, ( |
| 254 | + f"{missing} declare their own ARROW_SCHEMA but no WireRecordCase builds them. " |
| 255 | + "Add a case, or add the class to NOT_COVERED with the reason it needs none." |
| 256 | + ) |
| 257 | + |
| 258 | + # The excuse list has to stay honest too: an entry for a class that no |
| 259 | + # longer exists reads as coverage that does not exist. |
| 260 | + stale = sorted(set(NOT_COVERED) - set(classes)) |
| 261 | + assert not stale, f"NOT_COVERED lists {stale}, which no longer declare an ARROW_SCHEMA — drop them" |
| 262 | + |
| 263 | + both = sorted(set(NOT_COVERED) & set(CASES_BY_NAME)) |
| 264 | + assert not both, f"{both} are both covered by a case and excused in NOT_COVERED" |
0 commit comments