|
| 1 | +# Copyright 2025, 2026 Query Farm LLC - https://query.farm |
| 2 | + |
| 3 | +r"""Emit the VGI protocol's Arrow schemas as C#, for vgi-csharp's test project. |
| 4 | +
|
| 5 | +The companion of ``vgi.codegen.csharp_types``. That module emits the C# record |
| 6 | +types vgi-csharp derives its wire schemas from; this one emits the schemas the |
| 7 | +derivation must produce, taken from the protocol rather than from C#, so a test |
| 8 | +can compare the two field for field — the same "second description" vgi-java's |
| 9 | +``WireRecordSchemaConformanceTest`` relies on. |
| 10 | +
|
| 11 | +Generating the types makes them agree with the protocol only as far as the |
| 12 | +generator's CLR mapping agrees with vgi-rpc-csharp's ``SchemaDerivation``; this |
| 13 | +is what checks that. It also covers what is still written by hand: the flat |
| 14 | +parameters of ``IVgiService``'s methods, and which record each method returns. |
| 15 | +
|
| 16 | +Three maps, all keyed by protocol names: |
| 17 | +
|
| 18 | +- ``Records`` — every record ``csharp_types`` emits, by C# type name. |
| 19 | +- ``MethodResults`` — each unary method's result record schema (the record |
| 20 | + inside the ``result`` binary), by wire method name; methods returning nothing |
| 21 | + or raw IPC are absent. |
| 22 | +- ``MethodParams`` — each method's params batch schema, by wire method name. |
| 23 | +
|
| 24 | +.. code-block:: bash |
| 25 | +
|
| 26 | + uv run --project ~/Development/vgi-python python scripts/regen_generated.py |
| 27 | +""" |
| 28 | + |
| 29 | +from __future__ import annotations |
| 30 | + |
| 31 | +import argparse |
| 32 | +import io |
| 33 | +import sys |
| 34 | +from typing import TYPE_CHECKING, Any |
| 35 | + |
| 36 | +import pyarrow as pa |
| 37 | +from vgi_rpc.rpc._types import MethodType, rpc_methods # type: ignore[attr-defined] |
| 38 | + |
| 39 | +from vgi.codegen._common import GeneratorError, _resolve_inner_schema, provenance_comment |
| 40 | +from vgi.codegen.csharp_types import build_model |
| 41 | +from vgi.protocol import VgiProtocol |
| 42 | + |
| 43 | +if TYPE_CHECKING: |
| 44 | + from typing import TextIO |
| 45 | + |
| 46 | + |
| 47 | +GENERATOR_VERSION = "1" |
| 48 | + |
| 49 | +DEFAULT_NAMESPACE = "QueryFarm.Vgi.Tests.Generated" |
| 50 | + |
| 51 | +_SCALARS: list[tuple[pa.DataType, str]] = [ |
| 52 | + (pa.bool_(), "BooleanType.Default"), |
| 53 | + (pa.int8(), "Int8Type.Default"), |
| 54 | + (pa.int16(), "Int16Type.Default"), |
| 55 | + (pa.int32(), "Int32Type.Default"), |
| 56 | + (pa.int64(), "Int64Type.Default"), |
| 57 | + (pa.uint8(), "UInt8Type.Default"), |
| 58 | + (pa.uint16(), "UInt16Type.Default"), |
| 59 | + (pa.uint32(), "UInt32Type.Default"), |
| 60 | + (pa.uint64(), "UInt64Type.Default"), |
| 61 | + (pa.float32(), "FloatType.Default"), |
| 62 | + (pa.float64(), "DoubleType.Default"), |
| 63 | + (pa.string(), "StringType.Default"), |
| 64 | + (pa.large_string(), "LargeStringType.Default"), |
| 65 | + (pa.binary(), "BinaryType.Default"), |
| 66 | + (pa.large_binary(), "LargeBinaryType.Default"), |
| 67 | +] |
| 68 | + |
| 69 | +_UNITS = {"s": "Second", "ms": "Millisecond", "us": "Microsecond", "ns": "Nanosecond"} |
| 70 | + |
| 71 | + |
| 72 | +def _type(dtype: pa.DataType, origin: str, indent: str) -> str: |
| 73 | + for proto, expr in _SCALARS: |
| 74 | + if dtype.equals(proto): |
| 75 | + return expr |
| 76 | + if pa.types.is_dictionary(dtype): |
| 77 | + index = _type(dtype.index_type, f"{origin}[dict index]", indent) |
| 78 | + value = _type(dtype.value_type, f"{origin}[dict value]", indent) |
| 79 | + return f"new DictionaryType({index}, {value}, ordered: {str(dtype.ordered).lower()})" |
| 80 | + if pa.types.is_fixed_size_binary(dtype): |
| 81 | + return f"new FixedSizeBinaryType({dtype.byte_width})" |
| 82 | + if pa.types.is_timestamp(dtype): |
| 83 | + tz = "(string?)null" if dtype.tz is None else f'"{dtype.tz}"' |
| 84 | + return f"new TimestampType(TimeUnit.{_UNITS[dtype.unit]}, {tz})" |
| 85 | + if pa.types.is_list(dtype): |
| 86 | + return f"new ListType({_field(dtype.value_field, origin, indent)})" |
| 87 | + if pa.types.is_map(dtype): |
| 88 | + key = _field(dtype.key_field.with_name("key"), origin, indent) |
| 89 | + value = _field(dtype.item_field.with_name("value"), origin, indent) |
| 90 | + return f"new MapType({key}, {value})" |
| 91 | + if pa.types.is_struct(dtype): |
| 92 | + inner = indent + " " |
| 93 | + children = ",\n".join(inner + _field(dtype.field(i), origin, inner) for i in range(dtype.num_fields)) |
| 94 | + return f"new StructType(\n{indent}[\n{children},\n{indent}])" |
| 95 | + raise GeneratorError(f"vgi.codegen.csharp_schemas: unsupported Arrow type {dtype} at {origin}.") |
| 96 | + |
| 97 | + |
| 98 | +def _field(f: pa.Field[Any], origin: str, indent: str) -> str: |
| 99 | + return f'F("{f.name}", {_type(f.type, f"{origin}.{f.name}", indent)}, {str(f.nullable).lower()})' |
| 100 | + |
| 101 | + |
| 102 | +def _schema(schema: pa.Schema, origin: str) -> str: |
| 103 | + if len(schema) == 0: |
| 104 | + return "S()" |
| 105 | + indent = " " |
| 106 | + fields = ",\n".join(indent + _field(f, origin, indent) for f in schema) |
| 107 | + return f"S(\n{fields})" |
| 108 | + |
| 109 | + |
| 110 | +def _map(name: str, doc: str, entries: list[tuple[str, pa.Schema]]) -> str: |
| 111 | + lines = [ |
| 112 | + f" /// <summary>{doc}</summary>", |
| 113 | + f" public static IReadOnlyDictionary<string, Schema> {name} {{ get; }} = new Dictionary<string, Schema>", |
| 114 | + " {", |
| 115 | + ] |
| 116 | + for key, schema in entries: |
| 117 | + lines.append(f' ["{key}"] = {_schema(schema, key)},') |
| 118 | + lines.append(" };") |
| 119 | + return "\n".join(lines) + "\n" |
| 120 | + |
| 121 | + |
| 122 | +def emit(out: TextIO, *, namespace: str = DEFAULT_NAMESPACE) -> None: |
| 123 | + """Emit the generated C# schema class to *out*.""" |
| 124 | + model = build_model() |
| 125 | + records = [(name, model.records[name].schema) for name in sorted(model.records)] |
| 126 | + |
| 127 | + methods = rpc_methods(VgiProtocol) |
| 128 | + results: list[tuple[str, pa.Schema]] = [] |
| 129 | + params: list[tuple[str, pa.Schema]] = [] |
| 130 | + for method_name in sorted(methods): |
| 131 | + info = methods[method_name] |
| 132 | + params.append((method_name, info.params_schema)) |
| 133 | + if info.method_type == MethodType.UNARY and info.has_return: |
| 134 | + inner = _resolve_inner_schema(info.result_type, method_name) |
| 135 | + if inner is not None: |
| 136 | + results.append((method_name, inner)) |
| 137 | + |
| 138 | + body = io.StringIO() |
| 139 | + body.write("// Copyright 2025, 2026 Query Farm LLC - https://query.farm\n") |
| 140 | + body.write("// <auto-generated/>\n") |
| 141 | + body.write("\n") |
| 142 | + body.write("#nullable enable\n") |
| 143 | + body.write("\n") |
| 144 | + body.write("using Apache.Arrow;\n") |
| 145 | + body.write("using Apache.Arrow.Types;\n") |
| 146 | + body.write("\n") |
| 147 | + body.write(f"namespace {namespace};\n") |
| 148 | + body.write("\n") |
| 149 | + body.write( |
| 150 | + "/// <summary>\n" |
| 151 | + "/// The VGI protocol's Arrow schemas, generated from the protocol itself (vgi-python) rather\n" |
| 152 | + "/// than derived from this port's CLR types, so a test can compare the two. Dictionary ids and\n" |
| 153 | + "/// metadata are not part of the comparison.\n" |
| 154 | + "/// </summary>\n" |
| 155 | + ) |
| 156 | + body.write("public static class VgiProtocolSchemas\n{\n") |
| 157 | + body.write( |
| 158 | + " private static Schema S(params Field[] fields) => new(fields, metadata: null);\n\n" |
| 159 | + " private static Field F(string name, IArrowType type, bool nullable) => new(name, type, nullable);\n\n" |
| 160 | + ) |
| 161 | + body.write(_map("Records", "Every protocol record's schema, by its C# type name.", records)) |
| 162 | + body.write("\n") |
| 163 | + body.write( |
| 164 | + _map( |
| 165 | + "MethodResults", |
| 166 | + "Each unary method's result record schema, by wire method name. Methods that return " |
| 167 | + "nothing or raw IPC are absent.", |
| 168 | + results, |
| 169 | + ) |
| 170 | + ) |
| 171 | + body.write("\n") |
| 172 | + body.write(_map("MethodParams", "Each method's params batch schema, by wire method name.", params)) |
| 173 | + body.write("}\n") |
| 174 | + |
| 175 | + out.write( |
| 176 | + provenance_comment( |
| 177 | + generator_module="vgi.codegen.csharp_schemas", |
| 178 | + generator_command="python -m vgi.codegen.csharp_schemas", |
| 179 | + generator_version=GENERATOR_VERSION, |
| 180 | + regen_command_lines=[ |
| 181 | + "uv run --project ~/Development/vgi-python python scripts/regen_generated.py", |
| 182 | + ], |
| 183 | + body=body.getvalue(), |
| 184 | + ) |
| 185 | + ) |
| 186 | + out.write("\n") |
| 187 | + out.write(body.getvalue()) |
| 188 | + |
| 189 | + |
| 190 | +def main() -> None: |
| 191 | + """Console-script entrypoint — write the C# schema class to stdout.""" |
| 192 | + parser = argparse.ArgumentParser(description=__doc__) |
| 193 | + parser.add_argument("--namespace", default=DEFAULT_NAMESPACE, help="C# namespace of the emitted class") |
| 194 | + args = parser.parse_args() |
| 195 | + try: |
| 196 | + emit(sys.stdout, namespace=args.namespace) |
| 197 | + except GeneratorError as e: |
| 198 | + print(f"\nerror: {e}\n", file=sys.stderr) |
| 199 | + sys.exit(2) |
| 200 | + |
| 201 | + |
| 202 | +if __name__ == "__main__": |
| 203 | + main() |
0 commit comments