Classify native target failures separately (#177)

Author: Anionix

docs/specs/robustness-v1.md

@@ -35,6 +35,7 @@ Formalize the existing canonical verification as the conformance gate and add a
 - Run every bounded case through a fresh Python module subprocess with no shell interpolation.
 - Treat CSV, object JSONL, both Parquet settings, Lance base, and both Vortex settings as core targets.
 - Treat TsFile and the pinned FastLanes experiment as evidence-only targets.
+- Classify a non-zero exit from a project-seeded native target as `TARGET_FAILED`/`FAIL`; reserve `HARNESS_FAILED`/`INCOMPLETE` for runner, setup, or contract errors.
 - Cover row counts 0, 1, 1023, 1024, 1025, 2048, and 2049; dictionary cardinalities 1, 2, 255, and 256; NULL positions; UTF-8 and parser-sensitive strings; numeric boundaries; column-count errors; and truncation.
 - Add deterministic Arrow-table generation and empty, truncate, bit-flip, zero-range, append, and region mutation recipes.
 - Default full bounded runs to seed 20260703, 32 generated cases, 64 mutations per target, 30 seconds per case, and a 1 GiB artifact budget.

src/format_bench/model.py

@@ -46,6 +46,7 @@ class ObservedOutcome(StrEnum):
     TIMED_OUT = "TIMED_OUT"
     UNSUPPORTED = "UNSUPPORTED"
     BUDGET_EXHAUSTED = "BUDGET_EXHAUSTED"
+    TARGET_FAILED = "TARGET_FAILED"
     HARNESS_FAILED = "HARNESS_FAILED"
 
 

src/format_bench/robustness/native.py

@@ -346,6 +346,8 @@ def _case(
             observed = ObservedOutcome.CRASHED
         elif process["exit_code"] == 0:
             observed = ObservedOutcome.ACCEPTED
+        elif target.engine == "project-seeded":
+            observed = ObservedOutcome.TARGET_FAILED
         else:
             observed = ObservedOutcome.HARNESS_FAILED
         details: dict[str, object] = {

tests/test_model.py

@@ -66,6 +66,7 @@ def test_robustness_contract_exposes_the_public_vocabulary() -> None:
         ("MUST_NOT_CRASH", "REJECTED", "PASS"),
         ("MUST_NOT_CRASH", "CRASHED", "FAIL"),
         ("MUST_NOT_CRASH", "TIMED_OUT", "FAIL"),
+        ("MUST_NOT_CRASH", "TARGET_FAILED", "FAIL"),
         ("MUST_NOT_CRASH", "UNSUPPORTED", "INCOMPLETE"),
         ("MUST_NOT_CRASH", "BUDGET_EXHAUSTED", "INCOMPLETE"),
         ("MUST_NOT_CRASH", "HARNESS_FAILED", "INCOMPLETE"),

tests/test_robustness_native.py

@@ -213,6 +213,30 @@ def fake_process(command, cwd, timeout, output_budget_bytes):
     assert all(not item.startswith("-max_total_time=") for item in command)
 
 
+def test_native_suite_classifies_project_seeded_target_failure(
+    tmp_path: Path, monkeypatch
+) -> None:
+    root = Path(__file__).parents[1]
+    run_dir = tmp_path / "fastlanes-failed-run"
+    work_dir = tmp_path / "fastlanes"
+    _verified_fixture(root, run_dir)
+    _fake_target(work_dir / "build", "quick_fuzz_test", "print('target failed')")
+    target = replace(FASTLANES_NATIVE_TARGETS[0], work_dir=work_dir, source_commit=None)
+
+    def fake_process(command, cwd, timeout, output_budget_bytes):
+        return {"timed_out": False, "signal": None, "exit_code": 1}, "", "failed"
+
+    monkeypatch.setattr(native, "_process", fake_process)
+    result_path = run_native(
+        root, run_dir, duration_seconds=1, artifact_budget_mib=4, targets=(target,)
+    )
+
+    case = json.loads(result_path.read_text())["results"]["robustness_v1"]["cases"][0]
+    assert case["observed"] == ObservedOutcome.TARGET_FAILED.value
+    assert case["verdict"] == RobustnessVerdict.FAIL.value
+    assert case["process"]["exit_code"] == 1
+
+
 def test_native_suite_rejects_manifest_corpus_traversal(tmp_path: Path) -> None:
     root = Path(__file__).parents[1]
     run_dir = tmp_path / "run"