Skip to content

Commit 1d7be57

Browse files
committed
fix(#205): retire tautological host/embedded parity check
--backend host on the CLI/wasmtime harness tests/parity/ used can never diverge from --backend embedded: hasHostImport() is gated on builtin.target.cpu.arch == .wasm32 and root.is_embedded_engine, true only for the engine embedding targets, never the CLI. The CLI also never registers native_host_call_fn, so nativeCallRaw always returns .unsupported and every native call falls back to embedded regardless of the flag. The check was verifying embedded output equals embedded output; expanding its corpus as #205 originally asked would have reported false backend-divergence coverage rather than real signal. Removed: tests/parity/, test_runner.zig's runParity, the build.zig parity step, the redundant parity-test CI job, and stale references in CONTRIBUTING.md/Makefile/dev-docs. test remains the one real correctness gate. The actual gap this leaves — no test registers a genuine alternate implementation of the 8 CAP_* host-override capabilities and diffs it against embedded — is written up in dev-docs/roadmap.md for future work.
1 parent b5645b9 commit 1d7be57

9 files changed

Lines changed: 15 additions & 132 deletions

File tree

.githooks/pre-push

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ run git diff --check
2626

2727
run zig build -Dpreset=1m fuzz-native
2828
run zig build -Dpreset=1m test
29-
run zig build -Dpreset=1m parity
3029
run zig build -Dpreset=1m bench
3130
run zig build -Dpreset=1m native-cap
3231
run zig build -Dpreset=1m chaos

.github/workflows/ci.yml

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -198,34 +198,3 @@ jobs:
198198
echo "$PWD/binaryen-${BINARYEN_VERSION}/bin" >> "$GITHUB_PATH"
199199
- name: Run tests with heap paranoia (assert no live regions overwritten)
200200
run: zig build -Dpreset=dev -Dheap_paranoia=true test
201-
202-
parity-test:
203-
runs-on: ubuntu-latest
204-
steps:
205-
- uses: actions/checkout@v4
206-
- uses: mlugg/setup-zig@v2
207-
with:
208-
version: 0.16.0
209-
- name: Install wasmtime
210-
run: |
211-
set -euo pipefail
212-
TARBALL="wasmtime-${WASMTIME_VERSION}-x86_64-linux.tar.xz"
213-
URL="https://github.com/bytecodealliance/wasmtime/releases/download/${WASMTIME_VERSION}/${TARBALL}"
214-
for attempt in 1 2 3; do
215-
curl -fsSL --retry 3 --retry-delay 5 "$URL" -o "/tmp/${TARBALL}" && break
216-
[ "$attempt" -lt 3 ] && sleep 15 || exit 1
217-
done
218-
echo "${WASMTIME_SHA256} /tmp/${TARBALL}" | sha256sum --check --strict
219-
tar -xJf "/tmp/${TARBALL}" -C /tmp
220-
mkdir -p "$HOME/.wasmtime/bin"
221-
mv "/tmp/wasmtime-${WASMTIME_VERSION}-x86_64-linux/wasmtime" "$HOME/.wasmtime/bin/"
222-
echo "$HOME/.wasmtime/bin" >> "$GITHUB_PATH"
223-
- name: Install wasm-opt
224-
run: |
225-
TARBALL="binaryen-${BINARYEN_VERSION}-x86_64-linux.tar.gz"
226-
curl -fsSL "https://github.com/WebAssembly/binaryen/releases/download/${BINARYEN_VERSION}/${TARBALL}" -o "/tmp/${TARBALL}"
227-
echo "${BINARYEN_SHA256} /tmp/${TARBALL}" | sha256sum --check --strict
228-
tar -xzf "/tmp/${TARBALL}"
229-
echo "$PWD/binaryen-${BINARYEN_VERSION}/bin" >> "$GITHUB_PATH"
230-
- name: Run host/embedded backend parity tests
231-
run: zig build -Dpreset=dev parity

CONTRIBUTING.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,8 @@ If relevant, also run:
4949

5050
```bash
5151
make bench
52-
make parity
5352
```
5453

55-
`parity` checks that the native and WASM backends produce the same output. Run it when touching the VM or compiler.
56-
5754
## Profiling
5855

5956
Build the timing binary and profile with DWARF call graphs:

Makefile

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,3 @@ bench-release-stress:
7777
.PHONY: bench-recursion-stress
7878
bench-recursion-stress:
7979
GENGO_BENCH_INCLUDE_STRESS=1 GENGO_BENCH_FILTER='005_fib_recursive_35_stress.gengo' $(ZIG) build -Dpreset=stress -Dwasmtime=$(WASMTIME) bench-release
80-
81-
.PHONY: parity
82-
parity:
83-
$(ZIG) build -Dpreset=$(PRESET) -Dwasmtime=$(WASMTIME) parity

build.zig

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -488,14 +488,6 @@ pub fn build(b: *std.Build) void {
488488
const fuzz_gc_stress_step = b.step("fuzz-gc-stress", "Run fuzz tests under gc_stress (GC on every allocation)");
489489
fuzz_gc_stress_step.dependOn(&run_fuzz_gc_stress.step);
490490

491-
const run_parity = b.addRunArtifact(test_runner_exe);
492-
run_parity.step.dependOn(&install_test.step);
493-
run_parity.addArg("parity");
494-
run_parity.addArg(wasmtime_opt);
495-
run_parity.addArg("build/test/gengo-cli.wasm");
496-
const parity_step = b.step("parity", "Run host/embedded parity tests");
497-
parity_step.dependOn(&run_parity.step);
498-
499491
// ── Native CLI ────────────────────────────────────────────────────────────
500492

501493
const native_mod = b.createModule(.{

dev-docs/index.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ This directory holds contributor and project-internal documentation rather than
44

55
Contents:
66

7-
- `testing.md` — conformance, parity, and benchmark harnesses
7+
- `testing.md` — conformance and benchmark harnesses
88
- `roadmap.md` — current open items and exit criteria
99
- `opcodes.md` — VM opcode reference (fused-op internals; not public language semantics)
1010
- `design/compiler-architecture.md` — compiler conceptual reference: single-pass Pratt parser, expression/statement/declaration compilation, type system, scope/upvalues, module resolution

dev-docs/roadmap.md

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,19 @@ This becomes more relevant once the module system is used heavily. At that point
2424

2525
---
2626

27-
## 3. Exit Criteria
27+
## 3. Host-backend override parity has no real test coverage
28+
29+
**Status:** gap identified 2026-07-21, not scheduled.
30+
31+
The VM's `native_backend` policy (`--backend embedded`/`--backend host`) lets a host override built-in natives (`std.core.len`/`append`/`bytelen`, `std.conv.*`, `std.io.println` — the 8 `CAP_*` capabilities in `src/runtime/host_abi.zig`) with its own implementation, dispatched through `gengo_native_call`. No test ever registers a *real* alternate implementation and diffs its output against the embedded one.
32+
33+
`tests/parity/` (removed, see #205) ran the plain CLI/WASM binary under `wasmtime run` with `--backend host` — but `hasHostImport()` is gated on `builtin.target.cpu.arch == .wasm32 and root.is_embedded_engine`, true only for the `gengo-engine.wasm`/`libgengo-engine.so` embedding targets, never for the CLI. The CLI never registers `native_host_call_fn` either, so `nativeCallRaw` always returns `.unsupported` and every call falls back to the embedded path — `--backend host` on the CLI is architecturally a no-op, for any input, always. `engine_runner.zig`'s existing host-module tests only check that calling an *unregistered* host function produces the correct error; they don't register real implementations of the 8 override capabilities either.
34+
35+
Real coverage would need a native (or WASM component-model) test harness that registers a genuine `native_host_call_fn` implementing those 8 capabilities, runs scripts with `--backend host` against it, and diffs the result against both golden output and the embedded-backend result. That's new test infrastructure, not a corpus expansion.
36+
37+
---
38+
39+
## 4. Exit Criteria
2840

2941
1. All core capabilities are `done` and covered by conformance cases.
3042
2. Conformance suite runs in CI on every PR.

dev-docs/testing.md

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,6 @@ Notes:
2626
- Pass cases are in `tests/spec/*.gengo` with matching `.out` files.
2727
- Fail cases are in `tests/spec/fail/*.gengo` with matching `.err` token files.
2828

29-
## Parity Harness
30-
31-
Run embedded vs host-backend parity checks:
32-
33-
```bash
34-
zig build -Dpreset=1m parity
35-
```
36-
37-
Notes:
38-
- Parity cases live in `tests/parity/*.gengo`.
39-
- Host backend gracefully falls back to VM-local implementations when host import is unavailable.
40-
4129
## Bench Harness
4230

4331
Run benchmarks:

tools/test_runner.zig

Lines changed: 1 addition & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ pub fn main(init: std.process.Init.Minimal) !void {
2020
const args = argv_storage[0..arg_count];
2121

2222
if (args.len < 2) {
23-
std.debug.print("usage: test-runner <conformance|bench|parity> [wasmtime] [wasm] [--filter <pattern>]\n", .{});
23+
std.debug.print("usage: test-runner <conformance|bench> [wasmtime] [wasm] [--filter <pattern>]\n", .{});
2424
std.debug.print(" or: test-runner <native-cap|chaos> [gengo] [--filter <pattern>]\n", .{});
2525
std.process.exit(1);
2626
}
@@ -69,8 +69,6 @@ pub fn main(init: std.process.Init.Minimal) !void {
6969
try runConformance(alloc, wasmtime, wasm_path);
7070
} else if (std.mem.eql(u8, mode, "bench")) {
7171
try runBench(alloc, wasmtime, wasm_path);
72-
} else if (std.mem.eql(u8, mode, "parity")) {
73-
try runParity(alloc, wasmtime, wasm_path);
7472
} else {
7573
std.debug.print("unknown mode: {s}\n", .{mode});
7674
std.process.exit(1);
@@ -303,74 +301,6 @@ fn runBench(alloc: std.mem.Allocator, wasmtime: []const u8, wasm_path: []const u
303301
std.debug.print("Bench OK: {d} cases\n", .{pass_count});
304302
}
305303

306-
// ── Parity ─────────────────────────────────────────────────────────────────
307-
308-
fn runParity(alloc: std.mem.Allocator, wasmtime: []const u8, wasm_path: []const u8) !void {
309-
const parity_dir = "tests/parity";
310-
311-
var cases_buf: [MaxCases][]const u8 = undefined;
312-
const case_count = collectGengoFiles(alloc, parity_dir, &cases_buf) catch |err| {
313-
std.debug.print("cannot scan parity dir: {s}\n", .{@errorName(err)});
314-
std.process.exit(1);
315-
};
316-
317-
var pass_count: usize = 0;
318-
var errors: usize = 0;
319-
320-
for (cases_buf[0..case_count]) |path| {
321-
defer alloc.free(path);
322-
const base = path[0 .. path.len - 6];
323-
const got_emb_path = try std.fmt.allocPrint(alloc, "{s}.embedded.got", .{base});
324-
defer alloc.free(got_emb_path);
325-
const got_host_path = try std.fmt.allocPrint(alloc, "{s}.host.got", .{base});
326-
defer alloc.free(got_host_path);
327-
328-
std.debug.print("[PARITY] {s}\n", .{path});
329-
330-
// Embedded backend
331-
const emb_extra = try std.fmt.allocPrint(alloc, "--backend embedded {s}", .{path});
332-
defer alloc.free(emb_extra);
333-
const emb_result = runWasmtimeExtra(alloc, wasmtime, wasm_path, emb_extra);
334-
const emb_data = emb_result[0];
335-
const emb_failed = emb_result[1];
336-
defer alloc.free(emb_data);
337-
338-
if (emb_failed) {
339-
std.debug.print("embedded backend failed: {s}\n", .{path});
340-
errors += 1;
341-
continue;
342-
}
343-
344-
// Host backend
345-
const host_extra = try std.fmt.allocPrint(alloc, "--backend host {s}", .{path});
346-
defer alloc.free(host_extra);
347-
const host_result = runWasmtimeExtra(alloc, wasmtime, wasm_path, host_extra);
348-
const host_data = host_result[0];
349-
const host_failed = host_result[1];
350-
defer alloc.free(host_data);
351-
352-
if (host_failed) {
353-
std.debug.print("host backend failed: {s}\n", .{path});
354-
errors += 1;
355-
continue;
356-
}
357-
358-
if (!std.mem.eql(u8, emb_data, host_data)) {
359-
std.debug.print("backend output mismatch: {s}\n", .{path});
360-
errors += 1;
361-
continue;
362-
}
363-
364-
pass_count += 1;
365-
}
366-
367-
if (errors != 0) {
368-
std.debug.print("Host parity FAILED: {d} pass, {d} errors\n", .{ pass_count, errors });
369-
std.process.exit(1);
370-
}
371-
std.debug.print("Host parity OK: {d} cases\n", .{pass_count});
372-
}
373-
374304
// ── Native capability lane ────────────────────────────────────────────────
375305

376306
fn runNativeCap(alloc: std.mem.Allocator, gengo: []const u8, filter: ?[]const u8) !void {

0 commit comments

Comments
 (0)