Regression - BUGFIX - Report a crashed golden-check run as a crash, not a tolerance failure

A case that crashed before producing results was counted in n_fail and
exited as 'outside their golden tolerance'. The crash now travels in its
own n_run_failed field and gets a crash-specific exit message.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: d-burg

regression-harness/regress.jl

@@ -268,6 +268,7 @@ function main(args=ARGS)
         n_failed = 0
         n_changed = 0
         n_golden_fail = 0
+        n_golden_crashed = 0
         n_untracked = 0
         n_checked = 0
         n_no_golden = 0
@@ -293,6 +294,7 @@ function main(args=ARGS)
             elseif opts.check
                 summary = report_golden_check(db, case_spec, resolved_refs[1].commit_hash)
                 n_golden_fail += summary.n_fail
+                n_golden_crashed += summary.n_run_failed
                 n_untracked += summary.n_untracked
                 has_golden(case_spec.name) ? (n_checked += 1) : (n_no_golden += 1)
             else
@@ -311,6 +313,10 @@ function main(args=ARGS)
             @error "$n_failed run(s) failed — see the reports above"
             exit(1)
         end
+        if n_golden_crashed > 0
+            @error "$n_golden_crashed case(s) crashed before producing results — see the reports above (a crash, not a tolerance failure)"
+            exit(1)
+        end
         if n_golden_fail > 0
             @error "$n_golden_fail quantity/quantities are outside their golden tolerance"
             exit(1)

regression-harness/src/golden.jl

@@ -311,9 +311,11 @@ end
 """
 Compare a case's fresh run against its golden file and print the report.
 
-Returns `(n_pass, n_fail, n_untracked, n_informational)`. Only gating classes can fail;
-`diagnostic` and `unconverged` quantities are shown with their deviation and marked as
-informational, so a reader sees them move without the run failing on them.
+Returns `(n_pass, n_fail, n_untracked, n_informational, n_run_failed)`. Only gating classes
+can fail; `diagnostic` and `unconverged` quantities are shown with their deviation and marked
+as informational, so a reader sees them move without the run failing on them. A run that
+crashed outright is counted in `n_run_failed`, never in `n_fail`, so the caller can exit with
+a crash message instead of a tolerance message.
 """
 function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::String)
     golden = load_golden(case_spec.name)
@@ -322,15 +324,15 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St
     if golden === nothing
         println("  No golden file at $(golden_path(case_spec.name)) — nothing to check against.")
         println("  Generate one with:  regress --update-golden --cases $(case_spec.name) --reason \"...\"")
-        return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0)
+        return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=0)
     end
 
     quantities = get_quantities(db, commit_hash, case_spec.name)
     info = get_run_info(db, commit_hash, case_spec.name)
     if info !== nothing && !info.success
         println("  RUN FAILED — nothing to compare (this is a crash, not a tolerance failure):")
         println("  $(_short_err(info.error_msg))")
-        return (n_pass=0, n_fail=1, n_untracked=0, n_informational=0)
+        return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=1)
     end
 
     rows = Vector{Vector{String}}()
@@ -419,7 +421,7 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St
     n_untracked > 0 && push!(parts, "$n_untracked untracked")
     println("Summary: ", join(parts, ", "))
     println()
-    return (n_pass=n_pass, n_fail=n_fail, n_untracked=n_untracked, n_informational=n_informational)
+    return (n_pass=n_pass, n_fail=n_fail, n_untracked=n_untracked, n_informational=n_informational, n_run_failed=0)
 end
 
 """