Skip to content

Commit b2bb18c

Browse files
committed
test(cli): pin the index_repository exit contract
The contract landed measured but unpinned: six outcomes were checked by hand against the built binary and written into the pull request, and nothing in the suite would notice if a code moved. A verification that does not run again is a claim, not a test. Twelve cases over cbm_cli_index_exit_status, which grades a result envelope into a process exit code: clean run stays 0; an unusable file and a partial share above the ceiling both grade 2; "degraded" grades 2 on the status alone; an absent target grades 3 while a pipeline failure keeps 1 — the pair that shared code 1 before and gave the whole contract its reason. Three of them exist to fail if the grader stops working rather than to pass when it does: - the ceiling itself is exercised from both sides, 10 of 100 passing and 11 failing, because a test that only checks obvious excess stays green with the check switched off entirely; - the threshold overrides carry a positive control at the defaults, so "passed after raising the ceiling" cannot be confused with "no check ran"; - eight unreadable threshold values are pinned to fall back on the documented default, since a typo that silently disables a gate is the failure the strict parse was written to prevent. Silence is pinned too: a null, empty, unreadable, or content-less envelope preserves the base status in both directions. Inventing a quality verdict out of an unparsable payload would fail runs for the crime of an unexpected response shape. Envelopes are built with yyjson rather than hand-escaped, so the payload's quoting is the transport's and not the test's own. Signed-off-by: Vladislav Rahmanov <rahmanov.official@yandex.ru>
1 parent c7d6c72 commit b2bb18c

1 file changed

Lines changed: 285 additions & 0 deletions

File tree

‎tests/test_cli.c‎

Lines changed: 285 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14886,6 +14886,277 @@ TEST(cli_update_only_names_an_installer_that_exists_issue1632) {
1488614886
PASS();
1488714887
}
1488814888

14889+
/* ── index_repository exit contract ──────────────────────────────
14890+
*
14891+
* cbm_cli_index_exit_status grades a result envelope into a process exit
14892+
* code, because a freshness gate reads process status rather than a tool's
14893+
* account of itself. The six outcomes were measured by hand against the
14894+
* built binary when the contract landed; measured once is not pinned, and
14895+
* a code that moves silently is exactly what the contract exists to stop. */
14896+
14897+
/* Build the envelope the CLI actually receives: the payload travels as a
14898+
* JSON string inside content[0].text. Escaping it by hand in every test
14899+
* would put the test's own escaping on trial instead of the grader. */
14900+
static char *cli_index_envelope(const char *payload) {
14901+
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
14902+
if (!doc) {
14903+
return NULL;
14904+
}
14905+
yyjson_mut_val *root = yyjson_mut_obj(doc);
14906+
yyjson_mut_doc_set_root(doc, root);
14907+
yyjson_mut_val *content = yyjson_mut_arr(doc);
14908+
yyjson_mut_val *item = yyjson_mut_obj(doc);
14909+
yyjson_mut_obj_add_str(doc, item, "text", payload);
14910+
yyjson_mut_arr_append(content, item);
14911+
yyjson_mut_obj_add_val(doc, root, "content", content);
14912+
char *json = yyjson_mut_write(doc, 0, NULL);
14913+
yyjson_mut_doc_free(doc);
14914+
return json;
14915+
}
14916+
14917+
/* Grade one payload. Returns -1 only if the envelope could not be built, a
14918+
* value no exit code uses, so a setup failure cannot pass for a verdict. */
14919+
static int cli_index_grade(const char *payload, int base_status) {
14920+
char *envelope = cli_index_envelope(payload);
14921+
if (!envelope) {
14922+
return -1;
14923+
}
14924+
int status = cbm_cli_index_exit_status(envelope, base_status);
14925+
free(envelope);
14926+
return status;
14927+
}
14928+
14929+
typedef struct {
14930+
char *unusable;
14931+
char *partial_pct;
14932+
} cli_gate_env_t;
14933+
14934+
/* A test that means to exercise the documented defaults must not inherit
14935+
* whatever the developer happens to have exported. */
14936+
static cli_gate_env_t cli_gate_env_clear(void) {
14937+
cli_gate_env_t saved = {save_test_env("CBM_GATE_MAX_UNUSABLE"),
14938+
save_test_env("CBM_GATE_MAX_PARTIAL_PCT")};
14939+
cbm_unsetenv("CBM_GATE_MAX_UNUSABLE");
14940+
cbm_unsetenv("CBM_GATE_MAX_PARTIAL_PCT");
14941+
return saved;
14942+
}
14943+
14944+
static void cli_gate_env_restore(cli_gate_env_t saved) {
14945+
restore_test_env("CBM_GATE_MAX_UNUSABLE", saved.unusable);
14946+
restore_test_env("CBM_GATE_MAX_PARTIAL_PCT", saved.partial_pct);
14947+
}
14948+
14949+
/* A clean index keeps the code it always had: the contract adds verdicts,
14950+
* it does not make previously good runs start failing. */
14951+
TEST(cli_index_exit_clean_run_stays_zero) {
14952+
cli_gate_env_t saved = cli_gate_env_clear();
14953+
int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":4,"
14954+
"\"parse_partial_count\":0,\"parse_unusable_count\":0}",
14955+
CBM_CLI_EXIT_OK);
14956+
cli_gate_env_restore(saved);
14957+
ASSERT_EQ(status, CBM_CLI_EXIT_OK);
14958+
PASS();
14959+
}
14960+
14961+
/* A file that did not parse at all gets no tolerance — the default is zero,
14962+
* so one such file is already a quality failure. */
14963+
TEST(cli_index_exit_unusable_file_is_a_quality_failure) {
14964+
cli_gate_env_t saved = cli_gate_env_clear();
14965+
int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":5,"
14966+
"\"parse_partial_count\":0,\"parse_unusable_count\":1}",
14967+
CBM_CLI_EXIT_OK);
14968+
cli_gate_env_restore(saved);
14969+
ASSERT_EQ(status, CBM_CLI_EXIT_QUALITY);
14970+
PASS();
14971+
}
14972+
14973+
/* Partial parsing is graded as a share, not a count: 20 of 100 files is
14974+
* twice the default ceiling. */
14975+
TEST(cli_index_exit_partial_above_threshold_is_a_quality_failure) {
14976+
cli_gate_env_t saved = cli_gate_env_clear();
14977+
int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":100,"
14978+
"\"parse_partial_count\":20,\"parse_unusable_count\":0}",
14979+
CBM_CLI_EXIT_OK);
14980+
cli_gate_env_restore(saved);
14981+
ASSERT_EQ(status, CBM_CLI_EXIT_QUALITY);
14982+
PASS();
14983+
}
14984+
14985+
/* The boundary itself passes. The comparison is strict and integral on
14986+
* purpose: at exactly the documented ceiling a double could round either
14987+
* way, and a threshold whose verdict depends on rounding is not a
14988+
* threshold. 10 of 100 is the ceiling, and the ceiling is allowed. */
14989+
TEST(cli_index_exit_partial_exactly_at_threshold_passes) {
14990+
cli_gate_env_t saved = cli_gate_env_clear();
14991+
int at_ceiling = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":100,"
14992+
"\"parse_partial_count\":10,\"parse_unusable_count\":0}",
14993+
CBM_CLI_EXIT_OK);
14994+
/* One file more is over it, which proves the case above is the boundary
14995+
* and not simply a check that never fires. */
14996+
int over_ceiling = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":100,"
14997+
"\"parse_partial_count\":11,\"parse_unusable_count\":0}",
14998+
CBM_CLI_EXIT_OK);
14999+
cli_gate_env_restore(saved);
15000+
ASSERT_EQ(at_ceiling, CBM_CLI_EXIT_OK);
15001+
ASSERT_EQ(over_ceiling, CBM_CLI_EXIT_QUALITY);
15002+
PASS();
15003+
}
15004+
15005+
/* "degraded" is the pipeline's own verdict that the graph came out far
15006+
* smaller than the run expected. It carries no parse counts, so it has to
15007+
* be graded on the status alone. */
15008+
TEST(cli_index_exit_degraded_status_is_a_quality_failure) {
15009+
cli_gate_env_t saved = cli_gate_env_clear();
15010+
int status = cli_index_grade("{\"status\":\"degraded\",\"files_indexed\":40,"
15011+
"\"parse_partial_count\":0,\"parse_unusable_count\":0}",
15012+
CBM_CLI_EXIT_OK);
15013+
cli_gate_env_restore(saved);
15014+
ASSERT_EQ(status, CBM_CLI_EXIT_QUALITY);
15015+
PASS();
15016+
}
15017+
15018+
/* An absent target and a pipeline that fell over inside a reachable
15019+
* repository used to share code 1, which is the whole reason the contract
15020+
* was written: the caller could not tell "wrong path" from "broken run". */
15021+
TEST(cli_index_exit_separates_missing_target_from_broken_run) {
15022+
cli_gate_env_t saved = cli_gate_env_clear();
15023+
int target = cli_index_grade("{\"status\":\"error\",\"reason\":\"target_unavailable\"}",
15024+
CBM_CLI_EXIT_FAILURE);
15025+
int pipeline = cli_index_grade("{\"status\":\"error\",\"reason\":\"pipeline_failed\"}",
15026+
CBM_CLI_EXIT_FAILURE);
15027+
/* An error the mapping had not already flagged still has to land on a
15028+
* failure code rather than fall through as success. */
15029+
int unflagged = cli_index_grade("{\"status\":\"error\",\"reason\":\"pipeline_failed\"}",
15030+
CBM_CLI_EXIT_OK);
15031+
cli_gate_env_restore(saved);
15032+
ASSERT_EQ(target, CBM_CLI_EXIT_TARGET);
15033+
ASSERT_EQ(pipeline, CBM_CLI_EXIT_FAILURE);
15034+
ASSERT_EQ(unflagged, CBM_CLI_EXIT_FAILURE);
15035+
PASS();
15036+
}
15037+
15038+
/* Silence is never upgraded into a verdict. A payload the grader cannot
15039+
* read says nothing about index quality, and inventing a 2 from it would
15040+
* fail runs for the crime of an unexpected response shape. */
15041+
TEST(cli_index_exit_never_upgrades_silence) {
15042+
cli_gate_env_t saved = cli_gate_env_clear();
15043+
ASSERT_EQ(cbm_cli_index_exit_status(NULL, CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK);
15044+
ASSERT_EQ(cbm_cli_index_exit_status("", CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK);
15045+
ASSERT_EQ(cbm_cli_index_exit_status("not json at all", CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK);
15046+
/* A well-formed envelope carrying no content, and one whose text is not
15047+
* itself JSON — both are shapes a future response could take. */
15048+
ASSERT_EQ(cbm_cli_index_exit_status("{\"content\":[]}", CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK);
15049+
ASSERT_EQ(cli_index_grade("plain text, not a payload", CBM_CLI_EXIT_OK), CBM_CLI_EXIT_OK);
15050+
/* And the same shapes must not erase a failure already established. */
15051+
ASSERT_EQ(cbm_cli_index_exit_status(NULL, CBM_CLI_EXIT_FAILURE), CBM_CLI_EXIT_FAILURE);
15052+
cli_gate_env_restore(saved);
15053+
PASS();
15054+
}
15055+
15056+
/* Grading only ever makes a code more specific. A base failure survives a
15057+
* payload that looks perfectly healthy, because the transport already knew
15058+
* something the payload does not say. */
15059+
TEST(cli_index_exit_does_not_downgrade_a_failing_base) {
15060+
cli_gate_env_t saved = cli_gate_env_clear();
15061+
int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":9,"
15062+
"\"parse_partial_count\":0,\"parse_unusable_count\":0}",
15063+
CBM_CLI_EXIT_FAILURE);
15064+
cli_gate_env_restore(saved);
15065+
ASSERT_EQ(status, CBM_CLI_EXIT_FAILURE);
15066+
PASS();
15067+
}
15068+
15069+
/* Both thresholds are overridable, and both are read at grading time. */
15070+
TEST(cli_index_exit_thresholds_read_the_environment) {
15071+
cli_gate_env_t saved = cli_gate_env_clear();
15072+
const char *twenty_percent = "{\"status\":\"ok\",\"files_indexed\":100,"
15073+
"\"parse_partial_count\":20,\"parse_unusable_count\":0}";
15074+
const char *three_unusable = "{\"status\":\"ok\",\"files_indexed\":50,"
15075+
"\"parse_partial_count\":0,\"parse_unusable_count\":3}";
15076+
15077+
/* Positive control: at the defaults both payloads fail, so a pass below
15078+
* is the override working and not the check being absent. */
15079+
int partial_default = cli_index_grade(twenty_percent, CBM_CLI_EXIT_OK);
15080+
int unusable_default = cli_index_grade(three_unusable, CBM_CLI_EXIT_OK);
15081+
15082+
cbm_setenv("CBM_GATE_MAX_PARTIAL_PCT", "50", 1);
15083+
cbm_setenv("CBM_GATE_MAX_UNUSABLE", "5", 1);
15084+
int partial_raised = cli_index_grade(twenty_percent, CBM_CLI_EXIT_OK);
15085+
int unusable_raised = cli_index_grade(three_unusable, CBM_CLI_EXIT_OK);
15086+
cli_gate_env_restore(saved);
15087+
15088+
ASSERT_EQ(partial_default, CBM_CLI_EXIT_QUALITY);
15089+
ASSERT_EQ(unusable_default, CBM_CLI_EXIT_QUALITY);
15090+
ASSERT_EQ(partial_raised, CBM_CLI_EXIT_OK);
15091+
ASSERT_EQ(unusable_raised, CBM_CLI_EXIT_OK);
15092+
PASS();
15093+
}
15094+
15095+
/* A negative ceiling switches its own check off. This is the documented
15096+
* escape hatch for a repository whose grammars are known to be thin. */
15097+
TEST(cli_index_exit_negative_threshold_disables_the_check) {
15098+
cli_gate_env_t saved = cli_gate_env_clear();
15099+
cbm_setenv("CBM_GATE_MAX_PARTIAL_PCT", "-1", 1);
15100+
cbm_setenv("CBM_GATE_MAX_UNUSABLE", "-1", 1);
15101+
int status = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":100,"
15102+
"\"parse_partial_count\":99,\"parse_unusable_count\":7}",
15103+
CBM_CLI_EXIT_OK);
15104+
/* Disabling the parse checks must not disable the pipeline's own
15105+
* verdict: "degraded" is not a threshold and has no off switch. */
15106+
int degraded = cli_index_grade("{\"status\":\"degraded\",\"files_indexed\":10,"
15107+
"\"parse_partial_count\":0,\"parse_unusable_count\":0}",
15108+
CBM_CLI_EXIT_OK);
15109+
cli_gate_env_restore(saved);
15110+
ASSERT_EQ(status, CBM_CLI_EXIT_OK);
15111+
ASSERT_EQ(degraded, CBM_CLI_EXIT_QUALITY);
15112+
PASS();
15113+
}
15114+
15115+
/* A typo must not silently disable a gate. Anything that is not a whole
15116+
* number falls back to the documented default, so "1O" (letter O) fails the
15117+
* run it would have failed anyway instead of quietly waving it through. */
15118+
TEST(cli_index_exit_unreadable_threshold_falls_back_to_default) {
15119+
cli_gate_env_t saved = cli_gate_env_clear();
15120+
const char *twenty_percent = "{\"status\":\"ok\",\"files_indexed\":100,"
15121+
"\"parse_partial_count\":20,\"parse_unusable_count\":0}";
15122+
const char *unreadable[] = {"abc", "1O", "10pct", " 10", "10 ", "", "1e1", "10.0"};
15123+
for (size_t i = 0; i < sizeof(unreadable) / sizeof(unreadable[0]); i++) {
15124+
cbm_setenv("CBM_GATE_MAX_PARTIAL_PCT", unreadable[i], 1);
15125+
int status = cli_index_grade(twenty_percent, CBM_CLI_EXIT_OK);
15126+
if (status != CBM_CLI_EXIT_QUALITY) {
15127+
printf(" unreadable threshold \"%s\" gave exit %d\n", unreadable[i], status);
15128+
}
15129+
ASSERT_EQ(status, CBM_CLI_EXIT_QUALITY);
15130+
}
15131+
/* A value that DOES read still takes effect, so the loop above is about
15132+
* unreadable text and not about the override being ignored outright. */
15133+
cbm_setenv("CBM_GATE_MAX_PARTIAL_PCT", "50", 1);
15134+
int readable = cli_index_grade(twenty_percent, CBM_CLI_EXIT_OK);
15135+
cli_gate_env_restore(saved);
15136+
ASSERT_EQ(readable, CBM_CLI_EXIT_OK);
15137+
PASS();
15138+
}
15139+
15140+
/* A share needs a denominator. Without files_indexed the percentage cannot
15141+
* be computed at all, and a run must not be failed on a number nobody
15142+
* could work out. */
15143+
TEST(cli_index_exit_partial_without_denominator_is_not_graded) {
15144+
cli_gate_env_t saved = cli_gate_env_clear();
15145+
int no_denominator = cli_index_grade("{\"status\":\"ok\",\"parse_partial_count\":20}",
15146+
CBM_CLI_EXIT_OK);
15147+
int zero_denominator = cli_index_grade("{\"status\":\"ok\",\"files_indexed\":0,"
15148+
"\"parse_partial_count\":20}",
15149+
CBM_CLI_EXIT_OK);
15150+
/* An unusable file is an absolute count and still grades without one. */
15151+
int unusable_still_graded = cli_index_grade("{\"status\":\"ok\",\"parse_unusable_count\":1}",
15152+
CBM_CLI_EXIT_OK);
15153+
cli_gate_env_restore(saved);
15154+
ASSERT_EQ(no_denominator, CBM_CLI_EXIT_OK);
15155+
ASSERT_EQ(zero_denominator, CBM_CLI_EXIT_OK);
15156+
ASSERT_EQ(unusable_still_graded, CBM_CLI_EXIT_QUALITY);
15157+
PASS();
15158+
}
15159+
1488915160
SUITE(cli) {
1489015161
if (!th_secure_runtime_parent_new(g_cli_suite_runtime_parent,
1489115162
sizeof(g_cli_suite_runtime_parent), "cli-suite")) {
@@ -15304,6 +15575,20 @@ SUITE(cli) {
1530415575
/* Stdin argument gate (#1359) */
1530515576
RUN_TEST(cli_zero_argument_tool_never_reads_stdin_issue1359);
1530615577
RUN_TEST(cli_stdin_args_gate_tracks_tool_schema_issue1359);
15578+
15579+
/* index_repository exit contract */
15580+
RUN_TEST(cli_index_exit_clean_run_stays_zero);
15581+
RUN_TEST(cli_index_exit_unusable_file_is_a_quality_failure);
15582+
RUN_TEST(cli_index_exit_partial_above_threshold_is_a_quality_failure);
15583+
RUN_TEST(cli_index_exit_partial_exactly_at_threshold_passes);
15584+
RUN_TEST(cli_index_exit_degraded_status_is_a_quality_failure);
15585+
RUN_TEST(cli_index_exit_separates_missing_target_from_broken_run);
15586+
RUN_TEST(cli_index_exit_never_upgrades_silence);
15587+
RUN_TEST(cli_index_exit_does_not_downgrade_a_failing_base);
15588+
RUN_TEST(cli_index_exit_thresholds_read_the_environment);
15589+
RUN_TEST(cli_index_exit_negative_threshold_disables_the_check);
15590+
RUN_TEST(cli_index_exit_unreadable_threshold_falls_back_to_default);
15591+
RUN_TEST(cli_index_exit_partial_without_denominator_is_not_graded);
1530715592
cbm_cli_set_activation_runtime_parent_for_test(NULL);
1530815593
test_rmdir_r(g_cli_suite_runtime_parent);
1530915594
g_cli_suite_runtime_parent[0] = '\0';

0 commit comments

Comments
 (0)