Skip to content

Commit 3f69fec

Browse files
rustyconoverclaude
andcommitted
feat(catalog): companion catalogs (attach_catalogs) + catalog-table scan branches
Protocol source-of-truth for the lakehouse-federation feature consumed by the DuckDB VGI extension. - AttachCatalogInfo dataclass + CatalogAttachResult.attach_catalogs (nested-IPC list, like settings/secret_types); registered in codegen EXTRA_RESPONSE_TYPES. - ScanBranch gains nullable source_catalog/source_schema/source_table; a catalog-table branch is function_name="" + source_table set. - ReadOnlyCatalogInterface.catalog_attach emits attach_catalogs. - Fixture vgi/_test_fixtures/companion.py (vgi-fixture-companion-worker). Regenerate cpp/go/ts schemas after this. Additive + backward-compatible. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent fa17d9e commit 3f69fec

6 files changed

Lines changed: 266 additions & 3 deletions

File tree

‎packages/vgi-fixtures/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ dependencies = ["vgi-python[test-fixtures-writable]", "vgi-python[duckdb]"]
3838

3939
[project.scripts]
4040
vgi-fixture-worker = "vgi._test_fixtures.worker:main"
41+
vgi-fixture-companion-worker = "vgi._test_fixtures.companion:main"
4142
vgi-fixture-http = "vgi._test_fixtures.http_server:main"
4243
vgi-fixture-versioned-worker = "vgi._test_fixtures.versioned:main"
4344
vgi-fixture-versioned-tables-worker = "vgi._test_fixtures.versioned_tables:main"

‎tests/catalog/test_serialization.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1110,7 +1110,7 @@ class TestArrowSchemaCorrectness:
11101110
def test_catalog_attach_result_schema(self) -> None:
11111111
"""Verify CatalogAttachResult Arrow schema."""
11121112
schema = CatalogAttachResult.ARROW_SCHEMA
1113-
assert len(schema) == 14
1113+
assert len(schema) == 15
11141114
assert schema.field("attach_opaque_data").type == pa.binary()
11151115
assert schema.field("supports_transactions").type == pa.bool_()
11161116
assert schema.field("supports_time_travel").type == pa.bool_()
@@ -1120,6 +1120,7 @@ def test_catalog_attach_result_schema(self) -> None:
11201120
assert schema.field("default_schema").type == pa.string()
11211121
assert schema.field("settings").type == pa.list_(pa.binary())
11221122
assert schema.field("secret_types").type == pa.list_(pa.binary())
1123+
assert schema.field("attach_catalogs").type == pa.list_(pa.binary())
11231124
assert schema.field("resolved_data_version").type == pa.string()
11241125
assert schema.field("resolved_implementation_version").type == pa.string()
11251126

‎vgi/_test_fixtures/companion.py‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
# Copyright 2025, 2026 Query Farm LLC - https://query.farm
2+
3+
"""Test fixture: a VGI catalog that advertises a companion catalog + a
4+
multi-branch table whose branches are catalog-table branches on that companion.
5+
6+
Exercises the lakehouse-federation feature end-to-end:
7+
8+
* ``catalog_attach`` advertises ``attach_catalogs=[AttachCatalogInfo(...)]`` so
9+
the C++ extension attaches a companion catalog (DuckLake / attached DuckDB /
10+
…) on the client at VGI-attach time.
11+
* the ``data.hot_cold`` table's two branches are **catalog-table branches**
12+
(``function_name=""`` + ``source_*``) that scan ``<alias>.main.events`` in the
13+
companion, split hot/cold by ``branch_filter``.
14+
15+
Config via env (so the test controls the companion location):
16+
17+
* ``VGI_TEST_COMPANION_TARGET`` — ATTACH target, e.g.
18+
``ducklake:sqlite:/scratch/lake.sqlite`` or ``/scratch/companion.duckdb``.
19+
When unset, no companion is advertised (the table still exists but binding a
20+
branch against the missing companion errors — used by the opt-out test).
21+
* ``VGI_TEST_COMPANION_ALIAS`` — companion alias (default ``vgi_companion``).
22+
* ``VGI_TEST_COMPANION_DBTYPE`` — explicit db_type (default: infer from target
23+
scheme; set ``duckdb`` for a plain ``.duckdb`` file so the scheme allowlist
24+
admits it).
25+
* ``VGI_TEST_COMPANION_HIDDEN`` — ``1`` to attach the companion hidden.
26+
27+
Run with::
28+
29+
VGI_TEST_COMPANION_TARGET=ducklake:sqlite:/tmp/lake.sqlite \
30+
vgi-fixture-companion-worker
31+
"""
32+
33+
from __future__ import annotations
34+
35+
import os
36+
37+
import pyarrow as pa
38+
39+
from vgi.catalog import (
40+
AttachCatalogInfo,
41+
Catalog,
42+
ReadOnlyCatalogInterface,
43+
ScanBranch,
44+
ScanBranchesResult,
45+
Schema,
46+
Table,
47+
)
48+
from vgi.worker import Worker
49+
50+
_ALIAS = os.environ.get("VGI_TEST_COMPANION_ALIAS", "vgi_companion")
51+
_TARGET = os.environ.get("VGI_TEST_COMPANION_TARGET", "")
52+
_DBTYPE = os.environ.get("VGI_TEST_COMPANION_DBTYPE", "")
53+
_HIDDEN = os.environ.get("VGI_TEST_COMPANION_HIDDEN", "") == "1"
54+
55+
# Optional SECOND companion that fails (disallowed scheme) AFTER the first has
56+
# already attached — exercises the extension's partial-failure cleanup (the
57+
# first companion must not leak). Enabled with VGI_TEST_COMPANION_POISON=1.
58+
_POISON = os.environ.get("VGI_TEST_COMPANION_POISON", "") == "1"
59+
60+
61+
def _cols(**fields: pa.DataType) -> pa.Schema:
62+
return pa.schema([pa.field(n, t, nullable=True) for n, t in fields.items()])
63+
64+
65+
_COMPANION_CATALOG = Catalog(
66+
name="companion",
67+
default_schema="data",
68+
comment="VGI companion-catalog federation fixture",
69+
schemas=[
70+
Schema(
71+
name="data",
72+
comment="Multi-branch tables backed by a companion catalog",
73+
tables=[
74+
Table(
75+
name="hot_cold",
76+
columns=_cols(id=pa.int64(), val=pa.string()),
77+
comment="Two catalog-table branches on the companion's events table (hot/cold via branch_filter)",
78+
),
79+
],
80+
),
81+
],
82+
)
83+
84+
85+
class CompanionCatalog(ReadOnlyCatalogInterface):
86+
"""Advertises a companion catalog + a catalog-table-branch multi-branch table."""
87+
88+
catalog = _COMPANION_CATALOG
89+
90+
# Advertised only when a target is configured; empty otherwise (opt-out /
91+
# missing-companion tests).
92+
attach_catalogs = (
93+
(
94+
[
95+
AttachCatalogInfo(
96+
alias=_ALIAS,
97+
target=_TARGET,
98+
db_type=_DBTYPE,
99+
required=True,
100+
hidden=_HIDDEN,
101+
)
102+
]
103+
+ (
104+
# Second entry with a disallowed scheme ⇒ the extension throws
105+
# AFTER the first attached ⇒ partial-failure cleanup must release
106+
# the first.
107+
[AttachCatalogInfo(alias="vgi_companion_poison", target="ftp://nope/x", required=True)]
108+
if _POISON
109+
else []
110+
)
111+
)
112+
if _TARGET
113+
else []
114+
)
115+
116+
def table_scan_branches_get(
117+
self,
118+
*,
119+
attach_opaque_data, # noqa: ANN001
120+
transaction_opaque_data, # noqa: ANN001
121+
schema_name: str,
122+
name: str,
123+
at_unit: str | None = None,
124+
at_value: str | None = None,
125+
) -> ScanBranchesResult:
126+
del attach_opaque_data, transaction_opaque_data, at_unit, at_value
127+
if schema_name.lower() == "data" and name.lower() == "hot_cold":
128+
# Both arms scan the SAME companion table with disjoint branch_filters.
129+
# A query predicate disjoint with an arm's filter prunes that arm's
130+
# companion scan from the plan entirely.
131+
def _arm(branch_filter: str) -> ScanBranch:
132+
return ScanBranch(
133+
function_name="",
134+
positional_arguments=[],
135+
named_arguments={},
136+
branch_filter=branch_filter,
137+
source_catalog=_ALIAS,
138+
source_schema="main",
139+
source_table="events",
140+
)
141+
142+
# DuckLake builds its scan from parquet_scan's bind, so the branch
143+
# needs parquet loaded before it binds. Declaring it here makes the
144+
# rewriter auto-load it (works even where extension autoload is off).
145+
return ScanBranchesResult(
146+
branches=[_arm("id < 100"), _arm("id >= 100")],
147+
required_extensions=["parquet"],
148+
)
149+
msg = f"Unknown multi-branch table: {schema_name}.{name}"
150+
raise ValueError(msg)
151+
152+
153+
class CompanionCatalogWorker(Worker):
154+
"""Worker serving :class:`CompanionCatalog`."""
155+
156+
catalog_interface = CompanionCatalog
157+
158+
159+
def main() -> None:
160+
"""Console-script entry point (vgi-fixture-companion-worker)."""
161+
CompanionCatalogWorker.main()
162+
163+
164+
if __name__ == "__main__":
165+
main()

‎vgi/catalog/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
CatalogAttachResult,
1212
CatalogDataVersionRelease,
1313
CatalogExample,
14+
AttachCatalogInfo,
1415
CatalogInfo,
1516
CatalogInterface,
1617
CatalogObject,
@@ -68,6 +69,7 @@
6869
"TableInfo",
6970
"ViewInfo",
7071
"FunctionInfo",
72+
"AttachCatalogInfo",
7173
"CopyFromFormatInfo",
7274
"ScanBranch",
7375
"ScanBranchesResult",

‎vgi/catalog/catalog_interface.py‎

Lines changed: 94 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,55 @@ class CatalogInfo(ArrowSerializableDataclass):
176176
source_url: str | None = None
177177

178178

179+
@dataclass(frozen=True)
180+
class AttachCatalogInfo(ArrowSerializableDataclass):
181+
"""A companion catalog the client should ATTACH when this VGI catalog attaches.
182+
183+
Advertised via :attr:`CatalogAttachResult.attach_catalogs`. The VGI DuckDB
184+
extension attaches each entry at VGI-attach time (into the client's
185+
DatabaseManager) so that multi-branch catalog-table branches — and direct
186+
user queries — can resolve tables in a companion lakehouse (DuckLake /
187+
iceberg / postgres / …) without the client hand-attaching anything.
188+
189+
Trust: companion attach is remote-influenced, so the extension applies a
190+
scheme allowlist and a never-clobber conflict policy (it will never replace
191+
a catalog it did not itself create). See ``docs/companion_catalogs.md``.
192+
193+
Attributes:
194+
alias: Catalog name to attach as. Also the ``source_catalog`` a
195+
catalog-table :class:`ScanBranch` references. **Namespace this by
196+
your catalog identity** (e.g. ``"acme_lake"``) so two workers don't
197+
both claim ``"lake"`` on the same client — collisions are rejected,
198+
never silently merged.
199+
target: The ATTACH target — a path or DSN, e.g.
200+
``"ducklake:sqlite:/data/meta.sqlite"`` or
201+
``"postgres:dbname=… host=…"``.
202+
db_type: DuckDB storage/db type (e.g. ``"ducklake"``, ``"postgres"``).
203+
Empty ⇒ the extension infers it from the ``target`` scheme prefix.
204+
options: Extra ATTACH options forwarded verbatim (e.g. DuckLake
205+
``DATA_PATH``). Keys are matched case-insensitively by DuckDB.
206+
hidden: When true, attach hidden (excluded from ``duckdb_databases()``);
207+
still resolvable by qualified name and by branches. Use for
208+
branch-only companions the user shouldn't see.
209+
required: When true, a failure to attach (unreachable / conflict) fails
210+
the whole VGI ATTACH loudly. When false, the failure is logged and
211+
skipped; branches referencing it then error at bind.
212+
secret_ref: Optional name of a credential to pre-resolve (via the
213+
catalog's Orchard secret provider) and inject into the companion's
214+
ATTACH options — for metadata connections (e.g. a postgres DSN)
215+
where the catch-all path-keyed secret lookup isn't enough. Empty ⇒
216+
rely on the automatic catch-all provider for data-file creds.
217+
"""
218+
219+
alias: str
220+
target: str
221+
db_type: str = ""
222+
options: dict[str, str] = field(default_factory=dict)
223+
hidden: bool = False
224+
required: bool = False
225+
secret_ref: str = ""
226+
227+
179228
@dataclass(frozen=True)
180229
class CatalogAttachResult(ArrowSerializableDataclass):
181230
"""Result from attaching to a catalog.
@@ -199,6 +248,11 @@ class CatalogAttachResult(ArrowSerializableDataclass):
199248
Each ExtensionOption is serialized as bytes for Arrow compatibility.
200249
secret_types: Secret types registered with DuckDB's SecretManager. Each
201250
SecretTypeSpec is serialized as bytes for Arrow compatibility.
251+
attach_catalogs: Companion catalogs the client should ATTACH alongside
252+
this VGI catalog (lakehouse federation). Each :class:`AttachCatalogInfo`
253+
is serialized as bytes for Arrow compatibility. Attached at
254+
VGI-attach time; detached (refcounted) on DETACH. See
255+
``AttachCatalogInfo`` and ``docs/companion_catalogs.md``.
202256
comment: Optional comment describing this catalog/database.
203257
tags: Optional key-value tags associated with this catalog/database.
204258
supports_column_statistics: Whether any tables in this catalog can
@@ -221,6 +275,7 @@ class CatalogAttachResult(ArrowSerializableDataclass):
221275
default_schema: str = "main"
222276
settings: list[bytes] = field(default_factory=list)
223277
secret_types: list[bytes] = field(default_factory=list)
278+
attach_catalogs: list[bytes] = field(default_factory=list)
224279
comment: str | None = None
225280
tags: dict[str, str] = field(default_factory=dict)
226281
supports_column_statistics: bool = False
@@ -858,11 +913,33 @@ def deserialize(cls, batch: pa.RecordBatch) -> Self:
858913
class ScanBranch:
859914
"""One physical source backing a multi-branch scan.
860915
916+
A branch is one of two **kinds**:
917+
918+
* **Function branch** (the default) — ``function_name`` names a DuckDB table
919+
function bound with ``positional_arguments``/``named_arguments``.
920+
* **Catalog-table branch** — ``function_name`` is empty (``""``) and
921+
``source_table`` is set; the branch scans the base table
922+
``source_catalog.source_schema.source_table`` in an *attached* catalog
923+
(typically an :class:`AttachCatalogInfo` companion, e.g. a DuckLake
924+
table). The extension binds it via the catalog's own scan function, so a
925+
companion's snapshot/pruning semantics are honored.
926+
927+
The discriminator is simply "``source_table`` present ⇒ catalog-table kind".
928+
861929
Attributes:
862-
function_name: The DuckDB function to call for this branch
930+
function_name: The DuckDB function to call for a *function* branch
863931
(e.g., ``"read_parquet"``, ``"iceberg_scan"``, or a VGI
864932
table function). The C++ rewriter resolves this name against
865-
DuckDB's function catalog and binds it at optimize time.
933+
DuckDB's function catalog and binds it at optimize time. Empty
934+
string for a catalog-table branch.
935+
source_catalog: Catalog-table branch only — the attached catalog name
936+
(matches an :attr:`AttachCatalogInfo.alias`). ``None`` for function
937+
branches.
938+
source_schema: Catalog-table branch only — the schema of the source
939+
table. ``None`` for function branches.
940+
source_table: Catalog-table branch only — the base table name; its
941+
presence selects the catalog-table kind. ``None`` for function
942+
branches.
866943
positional_arguments: Positional arguments as PyArrow scalars,
867944
passed through to the function's ``bind``.
868945
named_arguments: Named arguments as PyArrow scalars.
@@ -892,13 +969,19 @@ class ScanBranch:
892969
named_arguments: dict[str, pa.Scalar] # type: ignore[type-arg]
893970
branch_filter: str | None = None
894971
writable: bool = False
972+
source_catalog: str | None = None
973+
source_schema: str | None = None
974+
source_table: str | None = None
895975

896976
ARROW_SCHEMA: ClassVar[pa.Schema] = pa.schema(
897977
[
898978
pa.field("function_name", pa.string(), nullable=False),
899979
pa.field("arguments", pa.binary(), nullable=False),
900980
pa.field("branch_filter", pa.string(), nullable=True),
901981
pa.field("writable", pa.bool_(), nullable=False),
982+
pa.field("source_catalog", pa.string(), nullable=True),
983+
pa.field("source_schema", pa.string(), nullable=True),
984+
pa.field("source_table", pa.string(), nullable=True),
902985
] # type: ignore[arg-type]
903986
)
904987

@@ -925,6 +1008,9 @@ def to_row_dict(self) -> dict[str, Any]:
9251008
"arguments": serialize_record_batch_bytes(argument_batch),
9261009
"branch_filter": self.branch_filter,
9271010
"writable": self.writable,
1011+
"source_catalog": self.source_catalog,
1012+
"source_schema": self.source_schema,
1013+
"source_table": self.source_table,
9281014
}
9291015

9301016
def serialize(self) -> bytes:
@@ -966,6 +1052,9 @@ def deserialize(cls, batch: pa.RecordBatch) -> Self:
9661052
branch_filter=cast("str | None", branch_filter_value) if branch_filter_value is not None else None,
9671053
# writable is non-nullable on the wire — trust the schema.
9681054
writable=bool(row["writable"]),
1055+
source_catalog=cast("str | None", row.get("source_catalog")),
1056+
source_schema=cast("str | None", row.get("source_schema")),
1057+
source_table=cast("str | None", row.get("source_table")),
9691058
)
9701059

9711060

@@ -2258,6 +2347,7 @@ class ReadOnlyCatalogInterface(CatalogInterface):
22582347
settings: list["SettingSpec"] = []
22592348
secret_types: list["SecretTypeSpec"] = []
22602349
attach_option_specs: list["AttachOptionSpec"] = []
2350+
attach_catalogs: list["AttachCatalogInfo"] = []
22612351

22622352
# NEW: Optional Catalog object for declarative definition
22632353
catalog: "Catalog | None" = None
@@ -2427,6 +2517,7 @@ def catalog_attach(
24272517
# Serialize settings and secret types for the attach result
24282518
serialized_settings = [s.serialize() for s in self.settings]
24292519
serialized_secret_types = [st.serialize() for st in self.secret_types]
2520+
serialized_attach_catalogs = [c.serialize_to_bytes() for c in self.attach_catalogs]
24302521

24312522
# Auto-derive supports_time_travel and supports_column_statistics from tables
24322523
self._build_registries()
@@ -2444,6 +2535,7 @@ def catalog_attach(
24442535
default_schema=self._default_schema_name,
24452536
settings=serialized_settings,
24462537
secret_types=serialized_secret_types,
2538+
attach_catalogs=serialized_attach_catalogs,
24472539
comment=self.catalog.comment if self.catalog is not None else None,
24482540
tags=dict(self.catalog.tags) if self.catalog is not None else {},
24492541
supports_column_statistics=has_column_statistics,

‎vgi/codegen/_common.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from vgi_rpc.rpc._types import MethodType, rpc_methods # type: ignore[attr-defined]
2424

2525
from vgi.catalog.catalog_interface import (
26+
AttachCatalogInfo,
2627
CatalogInfo,
2728
CatalogObject,
2829
CopyFromFormatInfo,
@@ -58,6 +59,7 @@
5859
ScanFunctionResult, # catalog_table_{scan,insert,update,delete}_function_get
5960
ScanBranchesResult, # catalog_table_scan_branches_get (top-level wrapper)
6061
ScanBranch, # one entry inside ScanBranchesResult.branches (binary blob)
62+
AttachCatalogInfo, # one entry inside CatalogAttachResult.attach_catalogs (binary blob)
6163
)
6264

6365

0 commit comments

Comments
 (0)