Skip to content

Commit 9fa54a7

Browse files
committed
test(own-cli): replay the frozen CLI contract with zero Python (#261 H.3)
`cargo test -p own-cli` builds the binary, runs it against every case in `tests/fixtures/cli_ownir/` and compares exit code, stdout bytes and stderr bytes. No Python runs — that separation is the point of the fixture, since a replay needing the reference would only prove the two agree on a machine that has both, which is not the machine the cutover is for. Three structural checks come before any byte comparison, because a fixture that has quietly stopped covering something passes every byte check it still has: manifest and case files must be the same set, the format version must agree on both sides, and every case must reproduce byte-for-byte on a second run. `<OS_ERROR>` is the only placeholder and it is held to rules of its own, asserted rather than trusted: it consumes the rest of ITS line, what it consumed must be non-empty, the `cannot read <path>: ` prefix stays byte-exact, and whatever the case expects after that line still has to match. The owen-convention text is checked through the process rather than the symbol: an integration test cannot link a `[[bin]]` crate, and comparing what a user actually sees against what the manifest carries is the stronger form anyway. The two failure-mode controls live behind the `fault-injection` feature. A catchable panic is asserted precisely — exit 70, exactly two stderr lines, the payload carried through, no findings, no backtrace unless OWNLANG_DEBUG — because the number is the contract. An uncatchable death is asserted only as a visible hard failure outside {0, 1, 2, 70} with nothing on stdout: #261 contracts no OS exit number for it, so none is asserted. Refs #261, #262, #250 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DTGMj7edRtjhyeXzkeiSWv
1 parent d471253 commit 9fa54a7

2 files changed

Lines changed: 483 additions & 0 deletions

File tree

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
//! The two forced-failure controls, MEASURED rather than asserted from the
2+
//! design — P-022 step 7b, #261.
3+
//!
4+
//! The whole file is behind the off-by-default `fault-injection` feature, so a
5+
//! production build compiles none of it and the binary it tests carries no
6+
//! hooks. Run it with:
7+
//!
8+
//! ```text
9+
//! cargo test -p own-cli --features fault-injection --test faults
10+
//! ```
11+
//!
12+
//! #261's two rulings, and the difference between them:
13+
//!
14+
//! ```text
15+
//! catchable panic -> exactly one actionable stderr diagnostic + exit 70,
16+
//! never 101. Asserted precisely, because the number is
17+
//! the contract: #262's launcher maps 70, and only 70,
18+
//! onto its internal-error path.
19+
//! uncatchable death -> a visible hard failure: non-zero, OUTSIDE
20+
//! {0, 1, 2, 70}, nothing on stdout, never masked. No
21+
//! particular OS exit number is contracted, so none is
22+
//! asserted — a signal on Unix and an abort status on
23+
//! Windows are both correct answers, and the number this
24+
//! run produced is recorded in the note instead.
25+
//! ```
26+
27+
#![cfg(feature = "fault-injection")]
28+
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
29+
30+
use std::path::PathBuf;
31+
use std::process::{Command, Output};
32+
33+
const FIXTURE_DIR: &str = concat!(
34+
env!("CARGO_MANIFEST_DIR"),
35+
"/../../../tests/fixtures/cli_ownir"
36+
);
37+
38+
/// A perfectly ordinary invocation — one that exits 0 without the hooks — so
39+
/// the only thing the control changes is the fault it injects.
40+
fn run(fault: &str, debug: bool) -> Output {
41+
let mut command = Command::new(env!("CARGO_BIN_EXE_own-cli"));
42+
command
43+
.args(["ownir", "../verdict_renders/render_empty.facts.json"])
44+
.current_dir(PathBuf::from(FIXTURE_DIR))
45+
.env_remove("OWNLANG_DEBUG")
46+
.env(fault, "1");
47+
if debug {
48+
command.env("OWNLANG_DEBUG", "1");
49+
}
50+
command.output().expect("own-cli runs")
51+
}
52+
53+
/// A catchable panic is exactly one diagnostic and exit 70 — never 101, never
54+
/// a finding, never an `ok` line.
55+
#[test]
56+
fn a_catchable_panic_is_one_diagnostic_and_exit_70() {
57+
let out = run("OWN_CLI_FAULT_PANIC", false);
58+
assert_eq!(
59+
out.status.code(),
60+
Some(70),
61+
"a panic must exit 70 (EX_SOFTWARE), never 101 — #262's launcher maps \
62+
only 70 onto its internal-error path"
63+
);
64+
assert!(
65+
out.stdout.is_empty(),
66+
"a panic must produce no findings and no `ok` line, got {:?}",
67+
String::from_utf8_lossy(&out.stdout)
68+
);
69+
let stderr = String::from_utf8_lossy(&out.stderr);
70+
let lines: Vec<&str> = stderr.lines().collect();
71+
assert_eq!(
72+
lines.len(),
73+
2,
74+
"EXACTLY one diagnostic (two lines), got: {stderr:?}"
75+
);
76+
assert!(
77+
lines[0].starts_with("own-cli: internal error: "),
78+
"the first line names the failure: {:?}",
79+
lines[0]
80+
);
81+
assert!(
82+
lines[0].contains("forced panic (fault-injection)"),
83+
"the payload is carried through, not replaced by a generic word: {:?}",
84+
lines[0]
85+
);
86+
assert!(
87+
lines[1].contains("This is a bug in the analyzer, not in your code"),
88+
"the second line tells the user it is not theirs: {:?}",
89+
lines[1]
90+
);
91+
assert!(
92+
!stderr.contains("note: run with `RUST_BACKTRACE"),
93+
"the hook must SUPPRESS the default panic output"
94+
);
95+
assert!(
96+
!stderr.contains("stack backtrace"),
97+
"no backtrace unless OWNLANG_DEBUG asks for one: {stderr:?}"
98+
);
99+
}
100+
101+
/// The debug half of the reference's asymmetry: the technical cause is shown
102+
/// and the exit code is kept. `run()` in `ownlang/__main__.py` does exactly
103+
/// this, and for exactly this reason — a re-raise would exit 1, which a caller
104+
/// reads as findings.
105+
#[test]
106+
fn debug_mode_shows_the_backtrace_and_still_exits_70() {
107+
let out = run("OWN_CLI_FAULT_PANIC", true);
108+
assert_eq!(
109+
out.status.code(),
110+
Some(70),
111+
"OWNLANG_DEBUG shows more; it must not change the exit code"
112+
);
113+
let stderr = String::from_utf8_lossy(&out.stderr);
114+
assert!(
115+
stderr.contains("forced panic (fault-injection)"),
116+
"the payload is printed: {stderr:?}"
117+
);
118+
assert!(
119+
stderr.contains("own-cli: panic:"),
120+
"the hook's debug line names the panic and its location: {stderr:?}"
121+
);
122+
assert!(
123+
out.stdout.is_empty(),
124+
"still no findings and no `ok` line under debug"
125+
);
126+
}
127+
128+
/// An uncatchable death is a VISIBLE hard failure. What is asserted is that it
129+
/// cannot be confused with any legal outcome; what is deliberately NOT asserted
130+
/// is the number, because #261 contracts no OS exit code for this case.
131+
#[test]
132+
fn an_uncatchable_death_is_a_visible_hard_failure() {
133+
let out = run("OWN_CLI_FAULT_ABORT", false);
134+
assert!(
135+
!out.status.success(),
136+
"an abort must never look like a clean run"
137+
);
138+
assert!(
139+
out.stdout.is_empty(),
140+
"an abort must produce no findings and no `ok` line, got {:?}",
141+
String::from_utf8_lossy(&out.stdout)
142+
);
143+
// `code()` is None when a signal killed the process (Unix); either way the
144+
// outcome must be outside the legal set, so a caller cannot read it as
145+
// clean, findings, a usage error or a handled internal error.
146+
if let Some(code) = out.status.code() {
147+
assert!(
148+
![0, 1, 2, 70].contains(&code),
149+
"an uncatchable death produced {code}, which is inside the legal \
150+
set {{0, 1, 2, 70}} — a caller would mistake it for an ordinary \
151+
outcome"
152+
);
153+
}
154+
}
155+
156+
/// The control has to be a control: without the environment variable the same
157+
/// invocation is an ordinary clean run, so the two tests above are measuring
158+
/// the fault rather than a broken binary.
159+
#[test]
160+
fn without_the_environment_variable_the_same_invocation_is_clean() {
161+
let out = Command::new(env!("CARGO_BIN_EXE_own-cli"))
162+
.args(["ownir", "../verdict_renders/render_empty.facts.json"])
163+
.current_dir(PathBuf::from(FIXTURE_DIR))
164+
.env_remove("OWNLANG_DEBUG")
165+
.env_remove("OWN_CLI_FAULT_PANIC")
166+
.env_remove("OWN_CLI_FAULT_ABORT")
167+
.output()
168+
.expect("own-cli runs");
169+
assert_eq!(out.status.code(), Some(0));
170+
assert!(String::from_utf8_lossy(&out.stdout).contains("0 findings."));
171+
}

0 commit comments

Comments
 (0)