Skip to content

Commit 33e9770

Browse files
rustyconoverclaude
andcommitted
feat(codegen): generate vgi-csharp's protocol types; release 0.39.0
vgi-csharp derives every wire schema from its CLR types by reflection, so its protocol records were the schemas -- and were kept in step with this repo by hand, which had already drifted (InitRequest/TableFunctionPlanRequest declared binary where the protocol says large_binary). - vgi.codegen.csharp_types emits every protocol record and wire enum as C# classes. Field order, Arrow types and nullability come from ARROW_SCHEMA; the annotation only picks among CLR types deriving the same Arrow type (enum vs dictionary-encoded string, nested class name). Shapes vgi-rpc-csharp cannot derive raise GeneratorError. C# names that differ from Python's are declared in CSHARP_NAMES. - vgi.codegen.csharp_schemas emits the protocol's schemas into vgi-csharp's test project, which compares what it actually serializes against them. - Both are regen_generated.py targets; tests/test_generated_csharp.py catches drift. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 8a2230b commit 33e9770

6 files changed

Lines changed: 1016 additions & 2 deletions

File tree

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "vgi-python"
3-
version = "0.38.1"
3+
version = "0.39.0"
44
description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
55
readme = "README.md"
66
keywords = [

‎scripts/regen_generated.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,12 @@
5555
("vgi.codegen.rust_request_builders", "vgi-protocol/src/generated/request_params.rs", "vgi-rust"),
5656
("vgi.codegen.ts_schemas", "src/generated/vgi-protocol-schemas.ts", "vgi-typescript"),
5757
("vgi.codegen.ts_client", "src/generated/vgi-client.ts", "vgi-typescript"),
58+
("vgi.codegen.csharp_types", "src/QueryFarm.Vgi/Protocol/Generated/VgiProtocolTypes.g.cs", "vgi-csharp"),
59+
(
60+
"vgi.codegen.csharp_schemas",
61+
"test/QueryFarm.Vgi.Tests/Generated/VgiProtocolSchemas.g.cs",
62+
"vgi-csharp",
63+
),
5864
(
5965
"vgi.codegen.java_schemas",
6066
"vgi/src/test/java/farm/query/vgi/generated/VgiProtocolSchemas.java",

‎tests/test_generated_csharp.py‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# Copyright 2025, 2026 Query Farm LLC - https://query.farm
2+
3+
"""Drift + behavior tests for `vgi.codegen.csharp_types` and `vgi.codegen.csharp_schemas`.
4+
5+
vgi-csharp checks in two generated files: the protocol's records and enums as C#
6+
types (its wire schemas are derived from them by reflection), and the protocol's
7+
schemas as C# Arrow schemas, which its test project compares that derivation
8+
against. Both must match what the generators emit right now.
9+
10+
The drift tests compare text, not parsed schemas: the types file is more than
11+
schemas (CLR types, attributes, initializers), and any byte of it changing is a
12+
change to what vgi-csharp compiles.
13+
14+
Skipped when the `vgi-csharp` checkout is not next to vgi-python;
15+
``VGI_CSHARP_ROOT`` overrides the location.
16+
"""
17+
18+
from __future__ import annotations
19+
20+
import io
21+
import os
22+
from pathlib import Path
23+
24+
import pytest
25+
26+
from vgi.codegen import csharp_schemas, csharp_types
27+
28+
_TYPES = Path("src/QueryFarm.Vgi/Protocol/Generated/VgiProtocolTypes.g.cs")
29+
_SCHEMAS = Path("test/QueryFarm.Vgi.Tests/Generated/VgiProtocolSchemas.g.cs")
30+
31+
_REGEN_HINT = (
32+
"To regenerate, run:\n"
33+
" uv run --project ~/Development/vgi-python python scripts/regen_generated.py\n"
34+
"\n"
35+
"Do not redirect a generator into its destination with `>`: the shell\n"
36+
"truncates the file before the generator runs, so any failure destroys\n"
37+
"the checked-in artifact. The script renders to memory first."
38+
)
39+
40+
41+
def _vgi_csharp_root() -> Path:
42+
override = os.environ.get("VGI_CSHARP_ROOT")
43+
if override:
44+
return Path(override)
45+
return Path(__file__).resolve().parents[2] / "vgi-csharp"
46+
47+
48+
def _render(module: object) -> str:
49+
buf = io.StringIO()
50+
module.emit(buf) # type: ignore[attr-defined]
51+
return buf.getvalue()
52+
53+
54+
@pytest.mark.parametrize("module", [csharp_types, csharp_schemas], ids=["types", "schemas"])
55+
def test_generator_is_deterministic(module: object) -> None:
56+
"""Calling emit() twice produces byte-identical output."""
57+
assert _render(module) == _render(module)
58+
59+
60+
@pytest.mark.parametrize(
61+
("module", "relative"),
62+
[(csharp_types, _TYPES), (csharp_schemas, _SCHEMAS)],
63+
ids=["types", "schemas"],
64+
)
65+
def test_checked_in_file_matches_generator(module: object, relative: Path) -> None:
66+
"""Drift check: the checked-in C# matches what the generator produces."""
67+
path = _vgi_csharp_root() / relative
68+
if not path.exists():
69+
pytest.skip(f"{path} not found; set VGI_CSHARP_ROOT or check out vgi-csharp")
70+
assert path.read_text() == _render(module), f"{path} is stale.\n{_REGEN_HINT}"
71+
72+
73+
def test_every_csharp_rename_is_used() -> None:
74+
"""A rename whose Python type no longer exists is dead, and would hide the next rename it needs."""
75+
model = csharp_types.build_model()
76+
emitted = set(model.records) | set(model.enums)
77+
reachable_python_names = {py for py, cs in csharp_types.CSHARP_NAMES.items() if cs in emitted}
78+
stale = set(csharp_types.CSHARP_NAMES) - reachable_python_names
79+
assert not stale, f"CSHARP_NAMES entries map to nothing emitted: {sorted(stale)}"
80+
81+
82+
def test_merged_items_responses_share_one_type() -> None:
83+
"""The catalog items responses collapse into one C# type, built from all of them."""
84+
record = csharp_types.build_model().records["ItemsResponse"]
85+
methods = {o.removeprefix("method '").split("'")[0] for o in record.origins}
86+
assert {"catalog_catalogs", "catalog_schemas", "catalog_table_get", "catalog_index_get"} <= methods
87+
assert [p.wire_name for p in record.props] == ["items"]
88+
89+
90+
def test_large_binary_list_carries_large_width() -> None:
91+
"""`list<large_binary>` is declared on the property; without it the derivation says `list<binary>`."""
92+
props = {p.wire_name: p for p in csharp_types.build_model().records["InitRequest"].props}
93+
assert "[global::QueryFarm.VgiRpc.Reflection.LargeWidth]" in props["join_keys"].attrs
94+
assert props["join_keys"].clr_type == "global::System.Collections.Generic.List<byte[]>?"
95+
assert "[global::QueryFarm.VgiRpc.Reflection.LargeWidth]" in props["pushdown_filters"].attrs
96+
97+
98+
def test_appended_nullable_flag_is_wire_optional() -> None:
99+
"""A nullable column typed as a plain value with a default is an appended column."""
100+
props = {p.wire_name: p for p in csharp_types.build_model().records["AttachOptionSpec"].props}
101+
assert props["required"].clr_type == "bool"
102+
assert "[global::QueryFarm.Vgi.Internal.WireOptional]" in props["required"].attrs
103+
104+
105+
def test_enum_members_round_trip_to_their_wire_names() -> None:
106+
"""C# derives an enum member's wire name as UPPER_SNAKE of its name; every member must survive that."""
107+
for e in csharp_types.build_model().enums.values():
108+
for m in e.members:
109+
derived = csharp_types._to_snake_case(m.name).upper()
110+
rendered = csharp_types._render_enum(e)
111+
assert derived == m.wire_name or f'RpcName("{m.wire_name}")' in rendered, (e.name, m.name)

‎uv.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vgi/codegen/csharp_schemas.py‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
# Copyright 2025, 2026 Query Farm LLC - https://query.farm
2+
3+
r"""Emit the VGI protocol's Arrow schemas as C#, for vgi-csharp's test project.
4+
5+
The companion of ``vgi.codegen.csharp_types``. That module emits the C# record
6+
types vgi-csharp derives its wire schemas from; this one emits the schemas the
7+
derivation must produce, taken from the protocol rather than from C#, so a test
8+
can compare the two field for field — the same "second description" vgi-java's
9+
``WireRecordSchemaConformanceTest`` relies on.
10+
11+
Generating the types makes them agree with the protocol only as far as the
12+
generator's CLR mapping agrees with vgi-rpc-csharp's ``SchemaDerivation``; this
13+
is what checks that. It also covers what is still written by hand: the flat
14+
parameters of ``IVgiService``'s methods, and which record each method returns.
15+
16+
Three maps, all keyed by protocol names:
17+
18+
- ``Records`` — every record ``csharp_types`` emits, by C# type name.
19+
- ``MethodResults`` — each unary method's result record schema (the record
20+
inside the ``result`` binary), by wire method name; methods returning nothing
21+
or raw IPC are absent.
22+
- ``MethodParams`` — each method's params batch schema, by wire method name.
23+
24+
.. code-block:: bash
25+
26+
uv run --project ~/Development/vgi-python python scripts/regen_generated.py
27+
"""
28+
29+
from __future__ import annotations
30+
31+
import argparse
32+
import io
33+
import sys
34+
from typing import TYPE_CHECKING, Any
35+
36+
import pyarrow as pa
37+
from vgi_rpc.rpc._types import MethodType, rpc_methods # type: ignore[attr-defined]
38+
39+
from vgi.codegen._common import GeneratorError, _resolve_inner_schema, provenance_comment
40+
from vgi.codegen.csharp_types import build_model
41+
from vgi.protocol import VgiProtocol
42+
43+
if TYPE_CHECKING:
44+
from typing import TextIO
45+
46+
47+
GENERATOR_VERSION = "1"
48+
49+
DEFAULT_NAMESPACE = "QueryFarm.Vgi.Tests.Generated"
50+
51+
_SCALARS: list[tuple[pa.DataType, str]] = [
52+
(pa.bool_(), "BooleanType.Default"),
53+
(pa.int8(), "Int8Type.Default"),
54+
(pa.int16(), "Int16Type.Default"),
55+
(pa.int32(), "Int32Type.Default"),
56+
(pa.int64(), "Int64Type.Default"),
57+
(pa.uint8(), "UInt8Type.Default"),
58+
(pa.uint16(), "UInt16Type.Default"),
59+
(pa.uint32(), "UInt32Type.Default"),
60+
(pa.uint64(), "UInt64Type.Default"),
61+
(pa.float32(), "FloatType.Default"),
62+
(pa.float64(), "DoubleType.Default"),
63+
(pa.string(), "StringType.Default"),
64+
(pa.large_string(), "LargeStringType.Default"),
65+
(pa.binary(), "BinaryType.Default"),
66+
(pa.large_binary(), "LargeBinaryType.Default"),
67+
]
68+
69+
_UNITS = {"s": "Second", "ms": "Millisecond", "us": "Microsecond", "ns": "Nanosecond"}
70+
71+
72+
def _type(dtype: pa.DataType, origin: str, indent: str) -> str:
73+
for proto, expr in _SCALARS:
74+
if dtype.equals(proto):
75+
return expr
76+
if pa.types.is_dictionary(dtype):
77+
index = _type(dtype.index_type, f"{origin}[dict index]", indent)
78+
value = _type(dtype.value_type, f"{origin}[dict value]", indent)
79+
return f"new DictionaryType({index}, {value}, ordered: {str(dtype.ordered).lower()})"
80+
if pa.types.is_fixed_size_binary(dtype):
81+
return f"new FixedSizeBinaryType({dtype.byte_width})"
82+
if pa.types.is_timestamp(dtype):
83+
tz = "(string?)null" if dtype.tz is None else f'"{dtype.tz}"'
84+
return f"new TimestampType(TimeUnit.{_UNITS[dtype.unit]}, {tz})"
85+
if pa.types.is_list(dtype):
86+
return f"new ListType({_field(dtype.value_field, origin, indent)})"
87+
if pa.types.is_map(dtype):
88+
key = _field(dtype.key_field.with_name("key"), origin, indent)
89+
value = _field(dtype.item_field.with_name("value"), origin, indent)
90+
return f"new MapType({key}, {value})"
91+
if pa.types.is_struct(dtype):
92+
inner = indent + " "
93+
children = ",\n".join(inner + _field(dtype.field(i), origin, inner) for i in range(dtype.num_fields))
94+
return f"new StructType(\n{indent}[\n{children},\n{indent}])"
95+
raise GeneratorError(f"vgi.codegen.csharp_schemas: unsupported Arrow type {dtype} at {origin}.")
96+
97+
98+
def _field(f: pa.Field[Any], origin: str, indent: str) -> str:
99+
return f'F("{f.name}", {_type(f.type, f"{origin}.{f.name}", indent)}, {str(f.nullable).lower()})'
100+
101+
102+
def _schema(schema: pa.Schema, origin: str) -> str:
103+
if len(schema) == 0:
104+
return "S()"
105+
indent = " "
106+
fields = ",\n".join(indent + _field(f, origin, indent) for f in schema)
107+
return f"S(\n{fields})"
108+
109+
110+
def _map(name: str, doc: str, entries: list[tuple[str, pa.Schema]]) -> str:
111+
lines = [
112+
f" /// <summary>{doc}</summary>",
113+
f" public static IReadOnlyDictionary<string, Schema> {name} {{ get; }} = new Dictionary<string, Schema>",
114+
" {",
115+
]
116+
for key, schema in entries:
117+
lines.append(f' ["{key}"] = {_schema(schema, key)},')
118+
lines.append(" };")
119+
return "\n".join(lines) + "\n"
120+
121+
122+
def emit(out: TextIO, *, namespace: str = DEFAULT_NAMESPACE) -> None:
123+
"""Emit the generated C# schema class to *out*."""
124+
model = build_model()
125+
records = [(name, model.records[name].schema) for name in sorted(model.records)]
126+
127+
methods = rpc_methods(VgiProtocol)
128+
results: list[tuple[str, pa.Schema]] = []
129+
params: list[tuple[str, pa.Schema]] = []
130+
for method_name in sorted(methods):
131+
info = methods[method_name]
132+
params.append((method_name, info.params_schema))
133+
if info.method_type == MethodType.UNARY and info.has_return:
134+
inner = _resolve_inner_schema(info.result_type, method_name)
135+
if inner is not None:
136+
results.append((method_name, inner))
137+
138+
body = io.StringIO()
139+
body.write("// Copyright 2025, 2026 Query Farm LLC - https://query.farm\n")
140+
body.write("// <auto-generated/>\n")
141+
body.write("\n")
142+
body.write("#nullable enable\n")
143+
body.write("\n")
144+
body.write("using Apache.Arrow;\n")
145+
body.write("using Apache.Arrow.Types;\n")
146+
body.write("\n")
147+
body.write(f"namespace {namespace};\n")
148+
body.write("\n")
149+
body.write(
150+
"/// <summary>\n"
151+
"/// The VGI protocol's Arrow schemas, generated from the protocol itself (vgi-python) rather\n"
152+
"/// than derived from this port's CLR types, so a test can compare the two. Dictionary ids and\n"
153+
"/// metadata are not part of the comparison.\n"
154+
"/// </summary>\n"
155+
)
156+
body.write("public static class VgiProtocolSchemas\n{\n")
157+
body.write(
158+
" private static Schema S(params Field[] fields) => new(fields, metadata: null);\n\n"
159+
" private static Field F(string name, IArrowType type, bool nullable) => new(name, type, nullable);\n\n"
160+
)
161+
body.write(_map("Records", "Every protocol record's schema, by its C# type name.", records))
162+
body.write("\n")
163+
body.write(
164+
_map(
165+
"MethodResults",
166+
"Each unary method's result record schema, by wire method name. Methods that return "
167+
"nothing or raw IPC are absent.",
168+
results,
169+
)
170+
)
171+
body.write("\n")
172+
body.write(_map("MethodParams", "Each method's params batch schema, by wire method name.", params))
173+
body.write("}\n")
174+
175+
out.write(
176+
provenance_comment(
177+
generator_module="vgi.codegen.csharp_schemas",
178+
generator_command="python -m vgi.codegen.csharp_schemas",
179+
generator_version=GENERATOR_VERSION,
180+
regen_command_lines=[
181+
"uv run --project ~/Development/vgi-python python scripts/regen_generated.py",
182+
],
183+
body=body.getvalue(),
184+
)
185+
)
186+
out.write("\n")
187+
out.write(body.getvalue())
188+
189+
190+
def main() -> None:
191+
"""Console-script entrypoint — write the C# schema class to stdout."""
192+
parser = argparse.ArgumentParser(description=__doc__)
193+
parser.add_argument("--namespace", default=DEFAULT_NAMESPACE, help="C# namespace of the emitted class")
194+
args = parser.parse_args()
195+
try:
196+
emit(sys.stdout, namespace=args.namespace)
197+
except GeneratorError as e:
198+
print(f"\nerror: {e}\n", file=sys.stderr)
199+
sys.exit(2)
200+
201+
202+
if __name__ == "__main__":
203+
main()

0 commit comments

Comments
 (0)