Skip to content

Commit 4c466fc

Browse files
committed
fix(build): make prod objects follow TEST_SEAMS; stop cx tmp leak
Two build/test-hygiene defects. 1. Prod vendored objects ignored TEST_SEAMS (and MEMWASTE flips) Why: CFLAGS_PROD carries $(TEST_SEAM_DEFINE), but the objects compiled once and linked into the product binary -- prod_lsp_all.o, the grammars, ts_runtime -- used GRAMMAR_CFLAGS without it. A TEST_SEAMS=1 binary therefore silently lacked the seam code inside lsp_all (ts_lsp.c budget probes), and as soon as cbm.c references a seam symbol defined in lsp_all -- the pending DeusData#1527 change adds cbm_lsp_work_steps in lsp/scope.c -- `make cbm TEST_SEAMS=1` fails to link, which breaks scripts/test.sh and the smoke/_test CI builds. Adding only the define is unsafe: none of these objects depended on $(BUILD_DIR)/.build-config, so flipping back to TEST_SEAMS=0 would link the stale seam-built lsp_all into a release binary. MEMWASTE had the same gap: its flags reach every vendored prod object through MEMWASTE_VENDOR_CFLAGS. Fix: GRAMMAR_CFLAGS carries $(TEST_SEAM_DEFINE) (as GRAMMAR_CFLAGS_TEST already carries the define), and every prod object whose flags take a BUILD_CONFIG_SIG variable depends on the .build-config stamp: prod_%.o, prod_ts_runtime.o, prod_lsp_all.o, prod_preprocessor.o, prod_lz4.o, prod_lz4hc.o, prod_zstd.o, prod_sqlite3.o, prod_tre.o. prod_mimalloc.o and the unixcoder blob take no config variable and stay off the stamp. Proof (macOS arm64, GNU Make 3.81): - DeusData#1527 tree, `make cbm TEST_SEAMS=1`: undefined _cbm_lsp_work_steps (3/3). With this Makefile it links and prod_lsp_all.o defines the symbol; dropping only the define brings the link error back. - main, TEST_SEAMS=1: prod_lsp_all.o had 0 seam symbols, now 1. - Flips 1 -> 0 -> 1 -> default each recompile the 169 stamped objects including prod_lsp_all.o; cbm_ts_lsp_test_budget_remaining in prod_lsp_all.o and in the binary is 1/0/1/0. A repeat build at the same config compiles and links nothing. The TEST_SEAMS=0 and default binaries are byte-identical to the pre-change release binary. - Revert-check: without prod_lsp_all.o's stamp prerequisite, a seams -> release flip keeps the seam object and the release binary carries the seam symbol (different sha256). - Release `make cbm` before/after: byte-identical binary (sha256 048e459dfbb4...), build shown reproducible from scratch. 2. The complexity suite leaked its temp corpora Why: cx_measure_pair and cx_measure_bigpkg_pair created /tmp/cbm_cx_base_*, cbm_cx_dbl_*, cbm_cxbig_a_* and cbm_cxbig_b_* and never removed them -- four dirs per run, more on failure retries. Fix: one cx_measure() helper builds both corpora, runs the pipeline on them and removes them with th_rmtree on success and on every failure path, touching only the dirs it created (the temp root is shared with concurrent runs). Only the metrics outlive the call; gates and counters are unchanged. Proof: /tmp listing diff around `build/c/test-runner complexity`: +4 dirs per run before (3/3), +0 after (3/3). Injected corpus-build failure, pipeline-run failure and assertion failure each leave +0. Suites: complexity 5/5 (0 leaked); TEST_SEAMS=1 build of test-runner and cbm, then complexity 5/5; make lint-ci passed. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 1f9b4db commit 4c466fc

2 files changed

Lines changed: 71 additions & 50 deletions

File tree

‎Makefile.cbm‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -830,8 +830,14 @@ BUILD_DIR = build/c
830830
# ── Object file compilation (grammars need relaxed warnings) ─────
831831

832832
# Grammar + tree-sitter runtime: compiled without -Werror (upstream code has warnings)
833+
# $(TEST_SEAM_DEFINE): these prod objects (grammars, ts_runtime, lsp_all) are
834+
# linked into the same binary as the CFLAGS_PROD sources, so both halves must
835+
# agree on the seam switch. lsp_all holds seam code of its own (ts_lsp.c's
836+
# budget probes, any seam counter a resolver defines): without the define a
837+
# TEST_SEAMS=1 binary silently lacks those seams, and a seam symbol that cbm.c
838+
# references fails to link. GRAMMAR_CFLAGS_TEST always carries the define.
833839
GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
834-
$(MEMWASTE_VENDOR_CFLAGS)
840+
$(TEST_SEAM_DEFINE) $(MEMWASTE_VENDOR_CFLAGS)
835841
GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
836842
-DCBM_ENABLE_TEST_SEAMS=1 $(SANITIZED_DEFINE) $(SANITIZE)
837843
GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
@@ -939,7 +945,9 @@ $(BUILD_DIR)/sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
939945
$(BUILD_DIR)/tsan_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
940946
$(CC) $(SQLITE3_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $<
941947

942-
$(BUILD_DIR)/prod_sqlite3.o: $(SQLITE3_SRC) Makefile.cbm | $(BUILD_DIR)
948+
# .build-config: SQLITE3_CFLAGS carries MEMWASTE_VENDOR_CFLAGS (see the stamp
949+
# under "Production binary").
950+
$(BUILD_DIR)/prod_sqlite3.o: $(SQLITE3_SRC) Makefile.cbm $(BUILD_DIR)/.build-config | $(BUILD_DIR)
943951
$(CC) $(SQLITE3_CFLAGS) -c -o $@ $<
944952

945953
# TRE regex (only compiled on Windows — POSIX uses system <regex.h>)
@@ -969,7 +977,7 @@ $(BUILD_DIR)/tre.o: $(TRE_SRC) | $(BUILD_DIR)
969977
$(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR)
970978
$(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $<
971979

972-
$(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR)
980+
$(BUILD_DIR)/prod_tre.o: $(TRE_SRC) $(BUILD_DIR)/.build-config | $(BUILD_DIR)
973981
$(CC) $(TRE_CFLAGS) -O2 $(MEMWASTE_VENDOR_CFLAGS) -c -o $@ $<
974982
endif
975983

@@ -1123,28 +1131,35 @@ TS_RUNTIME_OBJ_PROD = $(BUILD_DIR)/prod_ts_runtime.o
11231131
LSP_OBJ_PROD = $(BUILD_DIR)/prod_lsp_all.o
11241132
PP_OBJ_PROD = $(BUILD_DIR)/prod_preprocessor.o
11251133

1126-
$(BUILD_DIR)/prod_%.o: $(CBM_DIR)/%.c $(GRAMMAR_UNITY_DEPS) | $(BUILD_DIR)
1134+
# Every prod object whose flags carry a BUILD_CONFIG_SIG variable
1135+
# (TEST_SEAM_DEFINE via GRAMMAR_CFLAGS, MEMWASTE_VENDOR_CFLAGS) depends on the
1136+
# $(BUILD_DIR)/.build-config stamp. These objects are compiled once and linked
1137+
# into every later binary, so without the stamp a TEST_SEAMS or MEMWASTE flip
1138+
# relinks the PREVIOUS flavour: a release binary keeps seam code compiled into
1139+
# lsp_all, or a seams binary misses it. prod_mimalloc.o and the unixcoder blob
1140+
# take no config variable and stay off the stamp.
1141+
$(BUILD_DIR)/prod_%.o: $(CBM_DIR)/%.c $(GRAMMAR_UNITY_DEPS) $(BUILD_DIR)/.build-config | $(BUILD_DIR)
11271142
$(CC) $(GRAMMAR_CFLAGS) -c -o $@ $<
11281143

1129-
$(BUILD_DIR)/prod_ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR)
1144+
$(BUILD_DIR)/prod_ts_runtime.o: $(TS_RUNTIME_DEPS) $(BUILD_DIR)/.build-config | $(BUILD_DIR)
11301145
$(CC) $(GRAMMAR_CFLAGS) -c -o $@ $<
11311146

1132-
$(BUILD_DIR)/prod_lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
1147+
$(BUILD_DIR)/prod_lsp_all.o: $(LSP_UNITY_DEPS) $(BUILD_DIR)/.build-config | $(BUILD_DIR)
11331148
$(CC) $(GRAMMAR_CFLAGS) -c -o $@ $<
11341149

1135-
$(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
1150+
$(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp $(BUILD_DIR)/.build-config | $(BUILD_DIR)
11361151
$(CXX) $(CXXFLAGS_PROD) -w -I$(CBM_DIR)/vendored -c -o $@ $<
11371152

11381153
# Vendored LZ4 (compiled separately, not unity-built via lz4_store.c)
11391154
LZ4_OBJ_PROD = $(BUILD_DIR)/prod_lz4.o $(BUILD_DIR)/prod_lz4hc.o
1140-
$(BUILD_DIR)/prod_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
1155+
$(BUILD_DIR)/prod_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c $(BUILD_DIR)/.build-config | $(BUILD_DIR)
11411156
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR) -c -o $@ $<
1142-
$(BUILD_DIR)/prod_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
1157+
$(BUILD_DIR)/prod_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c $(BUILD_DIR)/.build-config | $(BUILD_DIR)
11431158
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
11441159

11451160
# Vendored zstd (compiled separately, not unity-built via zstd_store.c)
11461161
ZSTD_OBJ_PROD = $(BUILD_DIR)/prod_zstd.o
1147-
$(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
1162+
$(BUILD_DIR)/prod_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c $(BUILD_DIR)/.build-config | $(BUILD_DIR)
11481163
$(CC) -std=c11 -D_DEFAULT_SOURCE -O2 -w $(MEMWASTE_VENDOR_CFLAGS) -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
11491164

11501165
OBJS_VENDORED_PROD = $(MIMALLOC_OBJ_PROD) $(SQLITE3_OBJ_PROD) $(TRE_OBJ_PROD) $(GRAMMAR_OBJS_PROD) $(TS_RUNTIME_OBJ_PROD) $(LSP_OBJ_PROD) $(PP_OBJ_PROD) $(LZ4_OBJ_PROD) $(ZSTD_OBJ_PROD) $(UNIXCODER_OBJ)
@@ -1160,7 +1175,8 @@ MAIN_SRC = src/main.c
11601175
# configuration and it is rewritten only when that content changes, so ordinary
11611176
# incremental builds are unaffected. (Single-quoted: the signature may contain
11621177
# the escaped quotes of -DCBM_VERSION, which are literal inside '', but must not
1163-
# contain a literal single quote.)
1178+
# contain a literal single quote.) The config-flavoured prod objects above
1179+
# depend on the stamp for the same reason as the binary.
11641180
# SANITIZE belongs here: it changes the instrumentation compiled into every
11651181
# object, so a build that only changes it MUST rebuild. Its absence made
11661182
# `scripts/test.sh SANITIZE= ...` -- the documented way to get a plain build

‎tests/test_complexity.c‎

Lines changed: 44 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -433,36 +433,58 @@ static double cx_ratio(double num, double den) {
433433
return den > 0.0 ? num / den : 0.0;
434434
}
435435

436+
typedef int (*CxCorpusBuilder)(const char *root, int k);
437+
438+
static void cx_remove_corpus(const char *root) {
439+
if (th_rmtree(root) != 0) {
440+
fprintf(stderr, " [complexity] WARN: could not remove temp corpus %s\n", root);
441+
}
442+
}
443+
444+
/* Build one corpus shape at k and 2k in two fresh temp dirs, run the pipeline
445+
* on both, and remove both dirs before returning — on success and on every
446+
* failure path. Only the metrics outlive the call; the corpora used to stay
447+
* behind in the temp root, four per suite run. Only dirs this call created
448+
* are removed (the temp root is shared with concurrent runs). */
449+
static int cx_measure(CxCorpusBuilder build, const char *prefix_a, const char *prefix_b,
450+
CxMetrics *out_a, CxMetrics *out_b) {
451+
const char *tmp = cbm_tmpdir();
452+
char root_a[512];
453+
char root_b[512];
454+
snprintf(root_a, sizeof(root_a), "%s/%s_XXXXXX", tmp, prefix_a);
455+
snprintf(root_b, sizeof(root_b), "%s/%s_XXXXXX", tmp, prefix_b);
456+
bool made_a = cbm_mkdtemp(root_a) != NULL;
457+
bool made_b = made_a && cbm_mkdtemp(root_b) != NULL;
458+
int rc = -1;
459+
if (made_b && build(root_a, CX_K_BASE) == 0 && build(root_b, CX_K_BASE * 2) == 0) {
460+
char db_a[600];
461+
char db_b[600];
462+
snprintf(db_a, sizeof(db_a), "%s/cx.db", root_a);
463+
snprintf(db_b, sizeof(db_b), "%s/cx.db", root_b);
464+
if (cx_run(root_a, db_a, out_a) == 0 && cx_run(root_b, db_b, out_b) == 0) {
465+
rc = 0;
466+
}
467+
}
468+
if (made_b) {
469+
cx_remove_corpus(root_b);
470+
}
471+
if (made_a) {
472+
cx_remove_corpus(root_a);
473+
}
474+
return rc;
475+
}
476+
436477
/* Shared across the suite so the report test reuses the measured pair instead
437478
* of paying two more pipeline runs. */
438479
static CxMetrics g_cx_base;
439480
static CxMetrics g_cx_doubled;
440481
static bool g_cx_measured = false;
441-
static char g_cx_root_base[512];
442-
static char g_cx_root_doubled[512];
443482

444483
static int cx_measure_pair(void) {
445484
if (g_cx_measured) {
446485
return 0;
447486
}
448-
const char *tmp = cbm_tmpdir();
449-
snprintf(g_cx_root_base, sizeof(g_cx_root_base), "%s/cbm_cx_base_XXXXXX", tmp);
450-
snprintf(g_cx_root_doubled, sizeof(g_cx_root_doubled), "%s/cbm_cx_dbl_XXXXXX", tmp);
451-
if (!cbm_mkdtemp(g_cx_root_base) || !cbm_mkdtemp(g_cx_root_doubled)) {
452-
return -1;
453-
}
454-
if (cx_build_corpus(g_cx_root_base, CX_K_BASE) != 0 ||
455-
cx_build_corpus(g_cx_root_doubled, CX_K_BASE * 2) != 0) {
456-
return -1;
457-
}
458-
char db1[600];
459-
char db2[600];
460-
snprintf(db1, sizeof(db1), "%s/cx.db", g_cx_root_base);
461-
snprintf(db2, sizeof(db2), "%s/cx.db", g_cx_root_doubled);
462-
if (cx_run(g_cx_root_base, db1, &g_cx_base) != 0) {
463-
return -1;
464-
}
465-
if (cx_run(g_cx_root_doubled, db2, &g_cx_doubled) != 0) {
487+
if (cx_measure(cx_build_corpus, "cbm_cx_base", "cbm_cx_dbl", &g_cx_base, &g_cx_doubled) != 0) {
466488
return -1;
467489
}
468490
g_cx_measured = true;
@@ -553,25 +575,8 @@ static int cx_measure_bigpkg_pair(void) {
553575
if (g_cx_big_measured) {
554576
return 0;
555577
}
556-
const char *tmp = cbm_tmpdir();
557-
char ra[512];
558-
char rb[512];
559-
snprintf(ra, sizeof(ra), "%s/cbm_cxbig_a_XXXXXX", tmp);
560-
snprintf(rb, sizeof(rb), "%s/cbm_cxbig_b_XXXXXX", tmp);
561-
if (!cbm_mkdtemp(ra) || !cbm_mkdtemp(rb)) {
562-
return -1;
563-
}
564-
if (cx_build_bigpkg(ra, CX_K_BASE) != 0 || cx_build_bigpkg(rb, CX_K_BASE * 2) != 0) {
565-
return -1;
566-
}
567-
char db1[600];
568-
char db2[600];
569-
snprintf(db1, sizeof(db1), "%s/cx.db", ra);
570-
snprintf(db2, sizeof(db2), "%s/cx.db", rb);
571-
if (cx_run(ra, db1, &g_cx_big_base) != 0) {
572-
return -1;
573-
}
574-
if (cx_run(rb, db2, &g_cx_big_doubled) != 0) {
578+
if (cx_measure(cx_build_bigpkg, "cbm_cxbig_a", "cbm_cxbig_b", &g_cx_big_base,
579+
&g_cx_big_doubled) != 0) {
575580
return -1;
576581
}
577582
g_cx_big_measured = true;

0 commit comments

Comments
 (0)