From 26281c9e415f558dead7822eaeeb4d9b0c6a2e53 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Thu, 29 Oct 2026 17:40:03 +0100 Subject: [PATCH] fix(client-contract): keep vectors servable and judge every column by same_value The nested-cells vector's flags column holds one item type, the validator reports a vector column Arrow cannot build, and served_table judges flat columns by same_value as it does nested ones. --- .../rules/reading-order.json | 4 +- tools/tests/test_validate_client_contract.py | 59 ++++++++++++++++++- tools/validate_client_contract.py | 25 ++++---- 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/python/client-contract/src/dfo_client_contract/rules/reading-order.json b/python/client-contract/src/dfo_client_contract/rules/reading-order.json index bd1db59..a2a318f 100644 --- a/python/client-contract/src/dfo_client_contract/rules/reading-order.json +++ b/python/client-contract/src/dfo_client_contract/rules/reading-order.json @@ -1479,7 +1479,7 @@ 1.0 ], [ - 1 + false ] ] ] @@ -1497,7 +1497,7 @@ 2020, 4, [ - 1 + false ], { "b": 2, diff --git a/tools/tests/test_validate_client_contract.py b/tools/tests/test_validate_client_contract.py index ee02a33..3982480 100644 --- a/tools/tests/test_validate_client_contract.py +++ b/tools/tests/test_validate_client_contract.py @@ -21,6 +21,7 @@ import json import os import shutil +import struct import sys from collections.abc import Callable from pathlib import Path @@ -176,6 +177,15 @@ def _selector_query_with_cut(doc: Any) -> None: exchange["request"]["path"] += "&without_party_totals=true" +def _time_period_written_as_text(vector: Any) -> None: + """The second member serves its years as text, a valid Arrow column of + its own that mixes types with the first member's in the stacked table.""" + member = vector["members"][1] + at = member["columns"].index("time_period") + for row in member["rows"]: + row[at] = str(row[at]) + + def _vector(name: str, change: Callable[[Any], None]) -> Callable[[Any], None]: return lambda doc: change(next(v for v in doc["vectors"] if v["name"] == name)) @@ -1817,10 +1827,25 @@ def change(doc: Any) -> None: ( "rules/reading-order.json", _vector( - "ties-keep-served-order", - lambda v: v["members"][0]["rows"][0].__setitem__(4, "2022"), + "nested-cells-compare-by-value", + lambda v: v["members"][0]["rows"][1].__setitem__(4, [1]), + ), + "vector 'nested-cells-compare-by-value' member 0 column 'flags' is no " + "Arrow column", + ), + ( + "rules/reading-order.json", + _vector( + "nested-cells-compare-by-value", + lambda v: v["members"][0]["rows"][1].__setitem__(2, {"a": "x", "b": 2}), ), - "vector 'ties-keep-served-order' mixes JSON types in column", + "vector 'nested-cells-compare-by-value' member 0 column 'meta' is no " + "Arrow column", + ), + ( + "rules/reading-order.json", + _vector("stack-crs-and-dac2a", _time_period_written_as_text), + "vector 'stack-crs-and-dac2a' mixes JSON types in column", ), ( "scenarios/fetch-ref-expired-file-selector.json", @@ -2117,6 +2142,8 @@ def change(doc: Any) -> None: "reading-order-null-sorted-first", "reading-order-without-reading-columns", "reading-order-vector-row-ragged", + "reading-order-vector-list-items-mix-types", + "reading-order-vector-struct-field-mixes-types", "reading-order-vector-column-mixes-types", "selector-query-carries-the-cut", "unlisted-artifact-selector-query-carries-the-cut", @@ -2166,6 +2193,32 @@ def _parquet(table: pa.Table) -> bytes: return sink.getvalue().to_pybytes() +def _nan(payload: int) -> float: + return struct.unpack(" None: + """Cells `same_value` holds the same, which Arrow's `unique` tells apart, + are one value on both paths.""" + rows = [[2021 - i, "x", cell] for i, cell in enumerate(cells)] + columns = ["time_period", "label", "amount"] + vector = validator.table_from_rows(columns, rows) + served = validator.served_table( + _parquet( + pa.table({name: [row[i] for row in rows] for i, name in enumerate(columns)}) + ) + ) + assert "amount" in served.same + assert validator.column_groups([served], []) == validator.column_groups( + [vector], [] + ) + + def test_a_served_table_reads_list_struct_and_nan_columns() -> None: """Columns `unique` cannot hash are judged cell by cell, and a NaN column holds one value, in one table and over stacked ones.""" diff --git a/tools/validate_client_contract.py b/tools/validate_client_contract.py index debb55d..d9c1937 100644 --- a/tools/validate_client_contract.py +++ b/tools/validate_client_contract.py @@ -1906,17 +1906,13 @@ def served_table(data: bytes) -> ServedTable: nested = frozenset( field.name for field in table.schema if pa.types.is_nested(field.type) ) + # Every column is judged by `same_value`, as `table_from_rows` judges a + # vector's, so a vector and its Parquet file group their columns alike. same = {} for name in table.column_names if table.num_rows else (): - column = table.column(name) - # Arrow's `unique` counts NaN once and cannot hash a nested type, - # whose cells `same_value` compares. - if name in nested: - values = column.to_pylist() - if _one_value(values): - same[name] = values[0] - elif len(distinct := column.unique()) == 1: - same[name] = distinct[0].as_py() + values = table.column(name).to_pylist() + if _one_value(values): + same[name] = values[0] return ServedTable(tuple(table.column_names), table.num_rows, same, nested) @@ -3774,8 +3770,10 @@ def _json_type(value: Any) -> str: def _unorderable(members: Sequence[Mapping[str, Any]]) -> str | None: """Why a reading-order vector's members cannot be ordered, or None: a - row whose cells do not match its member's columns, or a column whose - cells, nulls aside, mix JSON types, which the rule gives no order.""" + row whose cells do not match its member's columns, a member's column + whose cells cannot form one Arrow column, as a served Parquet file holds + them, or a column whose cells, nulls aside, mix JSON types across the + stacked members, which the rule gives no order.""" types: dict[str, set[str]] = {} for index, member in enumerate(members): columns = member["columns"] @@ -3788,6 +3786,11 @@ def _unorderable(members: Sequence[Mapping[str, Any]]) -> str | None: for name, cell in zip(columns, row, strict=True): if cell is not None: types.setdefault(name, set()).add(_json_type(cell)) + for position, name in enumerate(columns): + try: + pa.array([row[position] for row in member["rows"]]) + except pa.ArrowException as exc: + return f"member {index} column {name!r} is no Arrow column: {exc}" mixed = sorted(name for name, found in types.items() if len(found) > 1) if mixed: return f"mixes JSON types in column {mixed[0]!r}"