fix(client-contract): keep vectors servable and judge every column by same_value (#34)

The nested-cells vector's flags column holds one item type, the validator reports a vector column Arrow cannot build, and served_table judges flat columns by same_value as it does nested ones.

Author: jm-rivera

python/client-contract/src/dfo_client_contract/rules/reading-order.json

@@ -1479,7 +1479,7 @@
                 1.0
               ],
               [
-                1
+                false
               ]
             ]
           ]
@@ -1497,7 +1497,7 @@
           2020,
           4,
           [
-            1
+            false
           ],
           {
             "b": 2,

tools/tests/test_validate_client_contract.py

@@ -21,6 +21,7 @@
 import json
 import os
 import shutil
+import struct
 import sys
 from collections.abc import Callable
 from pathlib import Path
@@ -176,6 +177,15 @@ def _selector_query_with_cut(doc: Any) -> None:
     exchange["request"]["path"] += "&without_party_totals=true"
 
 
+def _time_period_written_as_text(vector: Any) -> None:
+    """The second member serves its years as text, a valid Arrow column of
+    its own that mixes types with the first member's in the stacked table."""
+    member = vector["members"][1]
+    at = member["columns"].index("time_period")
+    for row in member["rows"]:
+        row[at] = str(row[at])
+
+
 def _vector(name: str, change: Callable[[Any], None]) -> Callable[[Any], None]:
     return lambda doc: change(next(v for v in doc["vectors"] if v["name"] == name))
 
@@ -1817,10 +1827,25 @@ def change(doc: Any) -> None:
         (
             "rules/reading-order.json",
             _vector(
-                "ties-keep-served-order",
-                lambda v: v["members"][0]["rows"][0].__setitem__(4, "2022"),
+                "nested-cells-compare-by-value",
+                lambda v: v["members"][0]["rows"][1].__setitem__(4, [1]),
+            ),
+            "vector 'nested-cells-compare-by-value' member 0 column 'flags' is no "
+            "Arrow column",
+        ),
+        (
+            "rules/reading-order.json",
+            _vector(
+                "nested-cells-compare-by-value",
+                lambda v: v["members"][0]["rows"][1].__setitem__(2, {"a": "x", "b": 2}),
             ),
-            "vector 'ties-keep-served-order' mixes JSON types in column",
+            "vector 'nested-cells-compare-by-value' member 0 column 'meta' is no "
+            "Arrow column",
+        ),
+        (
+            "rules/reading-order.json",
+            _vector("stack-crs-and-dac2a", _time_period_written_as_text),
+            "vector 'stack-crs-and-dac2a' mixes JSON types in column",
         ),
         (
             "scenarios/fetch-ref-expired-file-selector.json",
@@ -2117,6 +2142,8 @@ def change(doc: Any) -> None:
         "reading-order-null-sorted-first",
         "reading-order-without-reading-columns",
         "reading-order-vector-row-ragged",
+        "reading-order-vector-list-items-mix-types",
+        "reading-order-vector-struct-field-mixes-types",
         "reading-order-vector-column-mixes-types",
         "selector-query-carries-the-cut",
         "unlisted-artifact-selector-query-carries-the-cut",
@@ -2166,6 +2193,32 @@ def _parquet(table: pa.Table) -> bytes:
     return sink.getvalue().to_pybytes()
 
 
+def _nan(payload: int) -> float:
+    return struct.unpack("<d", struct.pack("<Q", 0x7FF8000000000000 | payload))[0]
+
+
+@pytest.mark.parametrize(
+    "cells",
+    [[0.0, -0.0], [_nan(0), _nan(1)]],
+    ids=["signed-zeros", "nan-payloads"],
+)
+def test_a_vector_and_its_parquet_file_group_columns_alike(cells: list[float]) -> None:
+    """Cells `same_value` holds the same, which Arrow's `unique` tells apart,
+    are one value on both paths."""
+    rows = [[2021 - i, "x", cell] for i, cell in enumerate(cells)]
+    columns = ["time_period", "label", "amount"]
+    vector = validator.table_from_rows(columns, rows)
+    served = validator.served_table(
+        _parquet(
+            pa.table({name: [row[i] for row in rows] for i, name in enumerate(columns)})
+        )
+    )
+    assert "amount" in served.same
+    assert validator.column_groups([served], []) == validator.column_groups(
+        [vector], []
+    )
+
+
 def test_a_served_table_reads_list_struct_and_nan_columns() -> None:
     """Columns `unique` cannot hash are judged cell by cell, and a NaN column
     holds one value, in one table and over stacked ones."""

tools/validate_client_contract.py

@@ -1906,17 +1906,13 @@ def served_table(data: bytes) -> ServedTable:
     nested = frozenset(
         field.name for field in table.schema if pa.types.is_nested(field.type)
     )
+    # Every column is judged by `same_value`, as `table_from_rows` judges a
+    # vector's, so a vector and its Parquet file group their columns alike.
     same = {}
     for name in table.column_names if table.num_rows else ():
-        column = table.column(name)
-        # Arrow's `unique` counts NaN once and cannot hash a nested type,
-        # whose cells `same_value` compares.
-        if name in nested:
-            values = column.to_pylist()
-            if _one_value(values):
-                same[name] = values[0]
-        elif len(distinct := column.unique()) == 1:
-            same[name] = distinct[0].as_py()
+        values = table.column(name).to_pylist()
+        if _one_value(values):
+            same[name] = values[0]
     return ServedTable(tuple(table.column_names), table.num_rows, same, nested)
 
 
@@ -3774,8 +3770,10 @@ def _json_type(value: Any) -> str:
 
 def _unorderable(members: Sequence[Mapping[str, Any]]) -> str | None:
     """Why a reading-order vector's members cannot be ordered, or None: a
-    row whose cells do not match its member's columns, or a column whose
-    cells, nulls aside, mix JSON types, which the rule gives no order."""
+    row whose cells do not match its member's columns, a member's column
+    whose cells cannot form one Arrow column, as a served Parquet file holds
+    them, or a column whose cells, nulls aside, mix JSON types across the
+    stacked members, which the rule gives no order."""
     types: dict[str, set[str]] = {}
     for index, member in enumerate(members):
         columns = member["columns"]
@@ -3788,6 +3786,11 @@ def _unorderable(members: Sequence[Mapping[str, Any]]) -> str | None:
             for name, cell in zip(columns, row, strict=True):
                 if cell is not None:
                     types.setdefault(name, set()).add(_json_type(cell))
+        for position, name in enumerate(columns):
+            try:
+                pa.array([row[position] for row in member["rows"]])
+            except pa.ArrowException as exc:
+                return f"member {index} column {name!r} is no Arrow column: {exc}"
     mixed = sorted(name for name, found in types.items() if len(found) > 1)
     if mixed:
         return f"mixes JSON types in column {mixed[0]!r}"