feat(catalog): unify required_field_filter_paths -> required_filters (AND-of-ORs)

Replace the flat, AND-only required_field_filter_paths with a single
required_filters field in conjunctive normal form: an AND (outer list) of
OR-groups (inner lists) of dotted-path column references. A group is satisfied
when any one of its paths has a filter; every group must be satisfied. So
(("accession_number",), ("ticker","cik")) means "accession_number AND one of
(ticker, cik)" — the ticker-OR-cik requirement the old flat list couldn't
express. Wire type becomes list<list<utf8>> (auto-derived from the annotation;
mirrors unique_constraints), kept as the trailing TableInfo field so older
extensions degrade gracefully to no-enforcement.

Updates the Table descriptor + validation, TableInfo, the rff_* fixtures
(singleton groups preserve prior AND behavior) plus a new rff_or OR-group
fixture, and the catalog tests. No backward compatibility (clean rename).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: rustyconover

tests/catalog/test_required_field_filter_paths.py

@@ -0,0 +1,92 @@
+"""Unit tests for the ``Table.required_filters`` field.
+
+The field is purely declarative on the Python side — enforcement runs in the C++
+optimizer extension. It is conjunctive normal form: an AND (outer tuple) of
+OR-groups (inner tuples) of dotted-path column references. These tests cover:
+
+1. Default-empty behaviour (no field set → ``TableInfo`` ships an empty list).
+2. Populated round-trip (singleton groups + a multi-path OR-group preserved).
+3. Validation: each path's leading dotted segment must be a real column.
+4. Empty-string paths and empty OR-groups are rejected loudly.
+"""
+
+from __future__ import annotations
+
+import pyarrow as pa
+import pytest
+
+from vgi.catalog.descriptors import Table
+
+
+class TestRequiredFiltersField:
+    """Behaviour of the declarative required_filters field."""
+
+    def _bbox_columns(self) -> pa.Schema:
+        """Return a small schema with a ``bbox`` STRUCT plus flat id/ticker."""
+        return pa.schema(
+            [
+                ("id", pa.int64()),
+                ("ticker", pa.string()),
+                (
+                    "bbox",
+                    pa.struct(
+                        [
+                            ("xmin", pa.float64()),
+                            ("ymin", pa.float64()),
+                            ("xmax", pa.float64()),
+                            ("ymax", pa.float64()),
+                        ]
+                    ),
+                ),
+            ]
+        )
+
+    def test_default_is_empty_tuple(self) -> None:
+        """Default is an empty tuple; TableInfo wire field is an empty list."""
+        t = Table(name="t", columns=self._bbox_columns())
+        assert t.required_filters == ()
+        info = t.to_table_info("main")
+        assert info.required_filters == []
+
+    def test_singleton_groups_round_trip(self) -> None:
+        """Singleton groups (the AND-only case) survive as a list of lists."""
+        groups = (("bbox.xmin",), ("bbox.xmax",), ("bbox.ymin",), ("bbox.ymax",))
+        t = Table(name="place", columns=self._bbox_columns(), required_filters=groups)
+        assert t.required_filters == groups
+        info = t.to_table_info("main")
+        assert info.required_filters == [list(g) for g in groups]
+
+    def test_or_group_round_trip(self) -> None:
+        """A multi-path OR-group survives alongside a singleton mandatory group."""
+        groups = (("id",), ("id", "ticker"))
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=groups)
+        info = t.to_table_info("main")
+        assert info.required_filters == [["id"], ["id", "ticker"]]
+
+    def test_top_level_path(self) -> None:
+        """A top-level column name (no dots) is a valid path."""
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=(("id",),))
+        assert t.to_table_info("main").required_filters == [["id"]]
+
+    def test_nested_path_is_not_unpacked_for_validation(self) -> None:
+        """Only the leading dotted segment is validated; deeper segments pass through."""
+        # Subfield validity isn't checked here (descriptor doesn't unpack struct
+        # schemas) — only the leading segment matters. Typos like ``bbox.nope``
+        # go through and DuckDB catches them at scan time.
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=(("bbox.nope",),))
+        assert t.required_filters == (("bbox.nope",),)
+
+    def test_unknown_leading_segment_raises(self) -> None:
+        """A path whose leading segment is not a real column is rejected."""
+        with pytest.raises(ValueError, match=r"unknown column 'nope'"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=(("nope.xmin",),))
+
+    def test_empty_string_path_raises(self) -> None:
+        """An empty string path is rejected loudly rather than silently accepted."""
+        with pytest.raises(ValueError, match="must not contain empty strings"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=(("",),))
+
+    def test_empty_group_raises(self) -> None:
+        """An empty OR-group is rejected loudly."""
+        with pytest.raises(ValueError, match="must not contain empty groups"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=((),))

tests/catalog/test_required_filters.py

@@ -1149,10 +1149,10 @@ def test_table_info_schema(self) -> None:
         assert schema.field("cardinality_estimate").type == pa.int64()
         assert schema.field("column_statistics").type == pa.binary()
         assert schema.field("bind_result").type == pa.binary()
-        assert schema.field("required_field_filter_paths").type == pa.list_(pa.string())
-        # required_field_filter_paths is the LAST field (positional schema
-        # compatibility: old C++ extensions ignore-by-position trailing fields).
-        assert schema.field(len(schema) - 1).name == "required_field_filter_paths"
+        assert schema.field("required_filters").type == pa.list_(pa.list_(pa.string()))
+        # required_filters is the LAST field (positional schema compatibility:
+        # old C++ extensions ignore-by-position trailing fields).
+        assert schema.field(len(schema) - 1).name == "required_filters"
 
     def test_view_info_schema(self) -> None:
         """Verify ViewInfo Arrow schema."""

tests/catalog/test_serialization.py

@@ -5,7 +5,7 @@
       "counts": {
         "functions": 183,
         "schemas": 2,
-        "tables": 58,
+        "tables": 59,
         "views": 3
       },
       "data_version_spec": null,
@@ -225,9 +225,14 @@
             },
             {
               "cols": 2,
-              "comment": "rff_none — control table with no required_field_filter_paths (opt-out fast path).",
+              "comment": "rff_none — control table with no required_filters (opt-out fast path).",
               "name": "rff_none"
             },
+            {
+              "cols": 2,
+              "comment": "rff_or — requires a filter on 'a' OR 'b' (a single OR-group).",
+              "name": "rff_or"
+            },
             {
               "cols": 2,
               "comment": "rff_parquet — native read_parquet delegation with bbox.* required filters.",
@@ -2316,13 +2321,13 @@
             },
             {
               "args": [],
-              "doc": "rff_nested — nested STRUCT(wrapper.mid.leaf) for required_field_filter_paths tests",
+              "doc": "rff_nested — nested STRUCT(wrapper.mid.leaf) for required_filters tests",
               "name": "rff_nested_scan",
               "type": "table"
             },
             {
               "args": [],
-              "doc": "rff_none — control table with no required_field_filter_paths",
+              "doc": "rff_none — control table with no required_filters",
               "name": "rff_none_scan",
               "type": "table"
             },
@@ -2334,13 +2339,13 @@
             },
             {
               "args": [],
-              "doc": "rff_simple — flat columns (a, b) for required_field_filter_paths tests",
+              "doc": "rff_simple — flat columns (a, b) for required_filters tests",
               "name": "rff_simple_scan",
               "type": "table"
             },
             {
               "args": [],
-              "doc": "rff_struct — STRUCT(s.a, s.b) + other for required_field_filter_paths tests",
+              "doc": "rff_struct — STRUCT(s.a, s.b) + other for required_filters tests",
               "name": "rff_struct_scan",
               "type": "table"
             },

tests/landing/describe.expected.json

@@ -3,7 +3,7 @@
 """Scan functions backing the ``rff_*`` required-filter sqllogictest Tables.
 
 Used by the ``vgi_required_filters_*.test`` matrix. These fixtures exercise
-the ``Table.required_field_filter_paths`` field +
+the ``Table.required_filters`` field +
 the C++ optimizer extension that enforces it. The five tables form a small
 matrix:
 
@@ -99,10 +99,10 @@
 )
 
 # rff_rowid — a row-id column (virtual, hidden from SELECT *) alongside a bbox
-# struct with required_field_filter_paths. A `WHERE rowid = N` predicate pushes
+# struct with required_filters. A `WHERE rowid = N` predicate pushes
 # a table_filter keyed by the COLUMN_IDENTIFIER_ROW_ID sentinel (>> column
 # count), which the optimizer's required-filter check must skip rather than
-# index out of bounds. See required_field_filter_paths_native.test.
+# index out of bounds. See required_filters_native.test.
 RFF_ROWID_COLUMNS = pa.schema(
     [
         pa.field("row_id", pa.int64(), metadata={b"is_row_id": b""}),
@@ -124,7 +124,7 @@
 
 RffSimpleScanFunction = _static_scan_function(
     func_name="rff_simple_scan",
-    func_description="rff_simple — flat columns (a, b) for required_field_filter_paths tests",
+    func_description="rff_simple — flat columns (a, b) for required_filters tests",
     output_schema=RFF_SIMPLE_COLUMNS,
     data={
         "a": [1, 2, 3],
@@ -134,7 +134,7 @@
 
 RffStructScanFunction = _static_scan_function(
     func_name="rff_struct_scan",
-    func_description="rff_struct — STRUCT(s.a, s.b) + other for required_field_filter_paths tests",
+    func_description="rff_struct — STRUCT(s.a, s.b) + other for required_filters tests",
     output_schema=RFF_STRUCT_COLUMNS,
     data={
         "s": [
@@ -148,7 +148,7 @@
 
 RffNestedScanFunction = _static_scan_function(
     func_name="rff_nested_scan",
-    func_description="rff_nested — nested STRUCT(wrapper.mid.leaf) for required_field_filter_paths tests",
+    func_description="rff_nested — nested STRUCT(wrapper.mid.leaf) for required_filters tests",
     output_schema=RFF_NESTED_COLUMNS,
     data={
         "wrapper": [
@@ -174,7 +174,7 @@
 
 RffNoneScanFunction = _static_scan_function(
     func_name="rff_none_scan",
-    func_description="rff_none — control table with no required_field_filter_paths",
+    func_description="rff_none — control table with no required_filters",
     output_schema=RFF_NONE_COLUMNS,
     data={
         "a": [1, 2, 3],