Skip to content

Commit c1da8fa

Browse files
rustyconoverclaude
andcommitted
test(copy-to): add header_repeat + fail_on_value writer fixture options
Extend the example_lines_out COPY-TO fixture to back the expanded shared SLT suite: - header_repeat (int, ge=0, le=3, default 1): when header=true, write the header line that many times — exercises ranged-option coercion + worker-side ge/le. - fail_on_value (str, default ''): write() raises mid-sink when a cell equals it — exercises the in-flight failure/teardown path. Ordered variant inherits both. Unit test updated for the 6-option set. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 6b51e61 commit c1da8fa

2 files changed

Lines changed: 34 additions & 6 deletions

File tree

‎tests/test_copy_to_function.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,4 +110,11 @@ def test_catalog_advertises_copy_to_format() -> None:
110110
assert fmt.comment == "Toy delimited-text writer for tests"
111111
assert fmt.tags.get("category") == "copy_to"
112112
opt_schema = pa.ipc.read_schema(pa.py_buffer(fmt.options))
113-
assert set(opt_schema.names) == {"delimiter", "null_string", "header", "on_exists"}
113+
assert set(opt_schema.names) == {
114+
"delimiter",
115+
"null_string",
116+
"header",
117+
"header_repeat",
118+
"on_exists",
119+
"fail_on_value",
120+
}

‎vgi/_test_fixtures/copy_to.py‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ class ExampleLinesCopyToArgs:
4242
null_string: Annotated[str, Arg("null_string", doc="Token written for SQL NULL")]
4343
delimiter: Annotated[str, Arg("delimiter", default=",", doc="Field separator")] = ","
4444
header: Annotated[bool, Arg("header", default=False, doc="Write a header row of column names")] = False
45+
header_repeat: Annotated[
46+
int,
47+
Arg("header_repeat", default=1, ge=0, le=3, doc="When header=true, write the header line this many times"),
48+
] = 1
4549
on_exists: Annotated[
4650
str,
4751
Arg(
@@ -51,6 +55,10 @@ class ExampleLinesCopyToArgs:
5155
doc="Behavior when the destination file already exists",
5256
),
5357
] = "overwrite"
58+
fail_on_value: Annotated[
59+
str,
60+
Arg("fail_on_value", default="", doc="If non-empty, fail mid-write when a cell equals this value"),
61+
] = ""
5462

5563

5664
class ExampleLinesCopyToFunction(CopyToFunction[ExampleLinesCopyToArgs]):
@@ -75,6 +83,13 @@ def write(
7583
params: TableBufferingParams[ExampleLinesCopyToArgs],
7684
) -> None:
7785
"""Buffer one input batch as an IPC blob in execution-scoped storage."""
86+
# Mid-sink failure trigger: raise during a process() call when a cell
87+
# matches fail_on_value. Exercises the in-flight teardown/recovery path.
88+
if options.fail_on_value:
89+
for col in batch.columns:
90+
for value in col.to_pylist():
91+
if value is not None and str(value) == options.fail_on_value:
92+
raise ValueError(f"example_lines_out: fail_on_value hit: {options.fail_on_value!r}")
7893
sink = pa.BufferOutputStream()
7994
with pa.ipc.new_stream(sink, batch.schema) as writer:
8095
writer.write_batch(batch)
@@ -100,24 +115,30 @@ def close(
100115
def fmt(value: object) -> str:
101116
return options.null_string if value is None else str(value)
102117

118+
def write_header(fh: object, names: list[str]) -> None:
119+
# header=true writes the column-name line `header_repeat` times.
120+
if options.header:
121+
for _ in range(options.header_repeat):
122+
fh.write(options.delimiter.join(names) + "\n") # type: ignore[attr-defined]
123+
103124
rows_written = 0
104125
with open(file_path, "w", encoding="utf-8") as fh:
105126
wrote_header = False
106127
for _log_id, blob in shards:
107128
table = pa.ipc.open_stream(blob).read_all()
108-
if options.header and not wrote_header:
109-
fh.write(options.delimiter.join(table.schema.names) + "\n")
129+
if not wrote_header:
130+
write_header(fh, list(table.schema.names))
110131
wrote_header = True
111132
for row in table.to_pylist():
112133
fh.write(options.delimiter.join(fmt(row[name]) for name in table.schema.names) + "\n")
113134
rows_written += 1
114-
# Empty COPY with header=true still emits the header row. We need the
135+
# Empty COPY with header=true still emits the header row(s). We need the
115136
# source column names; they ride the bind's input_schema.
116-
if options.header and not wrote_header:
137+
if not wrote_header:
117138
assert params.init_call is not None
118139
in_schema = params.init_call.bind_call.input_schema
119140
if in_schema is not None:
120-
fh.write(options.delimiter.join(in_schema.names) + "\n")
141+
write_header(fh, list(in_schema.names))
121142
return rows_written
122143

123144

0 commit comments

Comments
 (0)