Skip to content

Commit 6a61538

Browse files
committed
fix(ci): retrace changed vendor families and wrappers
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 2f10540 commit 6a61538

5 files changed

Lines changed: 160 additions & 12 deletions

File tree

‎scripts/build.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,8 @@ if [[ "$PHASE" != all ]] && $WITH_UI; then
169169
fi
170170

171171
# --retrace-from: each listed path becomes `make -W <path>` (treated as changed
172-
# now -- make's own dependency graph picks the objects; no timestamp race).
172+
# now -- the retrace helper supplies direct compilation entrypoints, so nested
173+
# includes do not depend on Makefile wildcard coverage; no timestamp race).
173174
RETRACE_ARGS=()
174175
if [[ -n "$RETRACE_FROM" ]]; then
175176
if [[ "$PHASE" != reuse || ! -f "$RETRACE_FROM" ]]; then

‎scripts/ci/codeql-retrace-vendored.sh‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
#!/usr/bin/env bash
2-
# codeql-retrace-vendored.sh — the vendored C/C++ files a pull request changes,
2+
# codeql-retrace-vendored.sh — prebuilt C/C++ translation units to retrace,
33
# for the PR c-cpp CodeQL scan (.github/workflows/codeql.yml).
44
#
55
# A PR's c-cpp scan compiles the third-party objects BEFORE CodeQL starts
66
# tracing (scripts/build.sh --vendored-only), so vendored C stays out of the
77
# analysis, while ALL first-party C is traced -- except the vendored files the
8-
# PR itself changes. This lists those; scripts/build.sh --reuse-vendored
9-
# --retrace-from <list> then rebuilds their objects under tracing (make -W, so
10-
# make's own dependency graph picks the objects).
8+
# PR itself changes, including grammar/runtime wrappers. This lists their
9+
# compilation entrypoints; scripts/build.sh --reuse-vendored --retrace-from
10+
# <list> then rebuilds their objects under tracing. Use direct prerequisites
11+
# for make -W: its shallow wildcard dependencies miss nested headers/sources,
12+
# and compiler include paths can contain aliases such as ../unix/prim.c.
1113
#
1214
# The changed files come from the PR merge commit (HEAD^1 is the base it was
1315
# merged onto, as in pr.yml's file-list fallback; needs fetch-depth 2) unless
@@ -16,7 +18,7 @@
1618
# the scan: the list is empty and the scan still covers all first-party C.
1719
#
1820
# Usage: scripts/ci/codeql-retrace-vendored.sh [--changed FILE] OUT
19-
# OUT is written with the vendored paths to retrace, one per line (may be
21+
# OUT is written with the entrypoints to retrace, one per line (may be
2022
# empty). The scope test is scripts/ci/select-lanes.sh --codeql-scope c-cpp.
2123
# Exit: 0 = list written · 2 = usage error.
2224
set -euo pipefail
@@ -78,12 +80,42 @@ fi
7880
printf '%s\n' "$changed" |
7981
bash "$ROOT/scripts/ci/select-lanes.sh" --codeql-scope c-cpp |
8082
while IFS= read -r path; do
83+
entrypoints=()
8184
case "$path" in
85+
internal/cbm/grammar_*.c | internal/cbm/ts_runtime.c)
86+
entrypoints=("$path")
87+
;;
88+
vendored/mimalloc/*) entrypoints=(vendored/mimalloc/src/static.c) ;;
89+
vendored/sqlite3/*) entrypoints=(vendored/sqlite3/sqlite3.c) ;;
90+
vendored/tre/*) entrypoints=(vendored/tre/tre_all.c) ;;
91+
internal/cbm/vendored/lz4/*)
92+
entrypoints=(internal/cbm/vendored/lz4/lz4.c internal/cbm/vendored/lz4/lz4hc.c)
93+
;;
94+
internal/cbm/vendored/zstd/*) entrypoints=(internal/cbm/vendored/zstd/zstd.c) ;;
95+
internal/cbm/vendored/ts_runtime/*) entrypoints=(internal/cbm/ts_runtime.c) ;;
96+
internal/cbm/vendored/grammars/*)
97+
family="${path#internal/cbm/vendored/grammars/}"
98+
wrapper="internal/cbm/grammar_${family%%/*}.c"
99+
if [ -f "$ROOT/$wrapper" ]; then
100+
entrypoints=("$wrapper")
101+
else
102+
# Shared or unfamiliar grammar code can have many consumers.
103+
entrypoints=("$ROOT"/internal/cbm/grammar_*.c)
104+
fi
105+
;;
106+
internal/cbm/vendored/common/*)
107+
entrypoints=("$ROOT"/internal/cbm/grammar_*.c)
108+
;;
82109
internal/cbm/vendored/* | vendored/*)
83-
if [ -f "$ROOT/$path" ]; then printf '%s\n' "$path"; fi
110+
# Other vendored code is compiled with the first-party build.
111+
entrypoints=("$path")
84112
;;
85113
esac
86-
done >"$OUT"
114+
for entrypoint in "${entrypoints[@]+"${entrypoints[@]}"}"; do
115+
entrypoint="${entrypoint#"$ROOT"/}"
116+
if [ -f "$ROOT/$entrypoint" ]; then printf '%s\n' "$entrypoint"; fi
117+
done
118+
done | LC_ALL=C sort -u >"$OUT"
87119
count=$(wc -l <"$OUT" | tr -d ' ')
88-
echo "vendored files to retrace: $count"
120+
echo "prebuilt translation units to retrace: $count"
89121
cat "$OUT"

‎scripts/test.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,8 @@ bash "$ROOT/tests/test_setup_scripts_contract.sh"
383383
# every September 2026 PR push and real failure against it.
384384
echo "=== Step 0z: PR lane selector contract ==="
385385
bash "$ROOT/tests/test_select_lanes.sh"
386+
echo "=== Step 0z1: CodeQL retraces affected prebuilt objects ==="
387+
bash "$ROOT/tests/test_codeql_retrace.sh"
386388
echo "=== Step 0z2: lane selector history replay ==="
387389
bash "$ROOT/scripts/test-impact/replay-selector.sh"
388390
echo "=== Step 0z3: lane-aware aggregate gates (ci-ok, shard union) ==="

‎tests/test_codeql_retrace.sh‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#!/usr/bin/env bash
2+
# Exercise the real production Make rules with tiny C fixtures. CodeQL starts
3+
# tracing after cbm-third-party, so every affected object must compile again,
4+
# while unrelated objects must remain reused. No dry-run/timestamp assertions.
5+
set -euo pipefail
6+
ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}"
7+
python3 - "$ROOT" "$BASH" <<'PY'
8+
import os
9+
import pathlib
10+
import re
11+
import subprocess
12+
import sys
13+
import tempfile
14+
15+
root, bash = pathlib.Path(sys.argv[1]), sys.argv[2]
16+
failures = []
17+
with tempfile.TemporaryDirectory(prefix="cbm-codeql-retrace-") as directory:
18+
work = pathlib.Path(directory)
19+
20+
def write(path, text):
21+
dest = work / path
22+
dest.parent.mkdir(parents=True, exist_ok=True)
23+
dest.write_text(text, encoding="utf-8")
24+
25+
for path in ("Makefile.cbm", "scripts/ci/codeql-retrace-vendored.sh",
26+
"scripts/ci/select-lanes.sh", ".github/codeql/codeql-config.yml"):
27+
write(path, (root / path).read_text(encoding="utf-8"))
28+
29+
# Nested and aliased includes deliberately escape the Makefile's shallow
30+
# wildcard dependencies. The compiler really compiles these translation units.
31+
write("vendored/mimalloc/src/static.c",
32+
'#include "mimalloc/internal.h"\n#include "prim/osx/prim.c"\n')
33+
write("vendored/mimalloc/include/mimalloc/internal.h", "#define MI_VALUE 7\n")
34+
write("vendored/mimalloc/src/prim/osx/prim.c", '#include "../unix/prim.c"\n')
35+
write("vendored/mimalloc/src/prim/unix/prim.c", "int mi_value(void) { return MI_VALUE; }\n")
36+
write("internal/cbm/vendored/common/scanner.h", "#define SHARED_VALUE 3\n")
37+
for language in ("go", "json"):
38+
write(f"internal/cbm/grammar_{language}.c",
39+
f'#include "vendored/grammars/{language}/parser.c"\n')
40+
write(f"internal/cbm/vendored/grammars/{language}/parser.c",
41+
'#include "nested/value.h"\n#include "../../common/scanner.h"\n'
42+
f'int grammar_{language}(void) {{ return VALUE + SHARED_VALUE; }}\n')
43+
write(f"internal/cbm/vendored/grammars/{language}/nested/value.h", "#define VALUE 2\n")
44+
write("internal/cbm/vendored/grammars/shared/shared.h", "/* future shared grammar helper */\n")
45+
write("internal/cbm/ts_runtime.c", '#include "vendored/ts_runtime/src/lib.c"\n')
46+
write("internal/cbm/vendored/ts_runtime/src/lib.c", "int runtime_value(void) { return 1; }\n")
47+
for path, symbol in (("vendored/sqlite3/sqlite3.c", "sqlite_value"),
48+
("vendored/tre/tre_all.c", "tre_value"),
49+
("internal/cbm/vendored/lz4/lz4.c", "lz4_value"),
50+
("internal/cbm/vendored/lz4/lz4hc.c", "lz4hc_value"),
51+
("internal/cbm/vendored/zstd/zstd.c", "zstd_value")):
52+
write(path, f"int {symbol}(void) {{ return 1; }}\n")
53+
write("internal/cbm/vendored/lz4/lz4.h", "/* shared LZ4 header */\n")
54+
write("vendored/sqlite3/sqlite3.h", "/* sqlite header */\n")
55+
write("internal/cbm/vendored/zstd/zstd.h", "/* zstd header */\n")
56+
write("vendored/nomic/code_vectors_blob.S", ".text\n")
57+
write("vendored/nomic/code_vectors.bin", "")
58+
write("src/main.c", "int main(void) { return 0; }\n")
59+
60+
def run(args):
61+
proc = subprocess.run(args, cwd=work, capture_output=True, text=True)
62+
if proc.returncode:
63+
raise RuntimeError(f"{args}: exit {proc.returncode}\n{proc.stdout}\n{proc.stderr}")
64+
return proc.stdout
65+
66+
make = ["make", "--no-print-directory", "-f", "Makefile.cbm", "cbm-third-party",
67+
"BUILD_DIR=build/c", "CC=" + os.environ.get("CC", "cc")]
68+
69+
def compiled(output):
70+
return set(re.findall(r" -c -o build/c/([^\s]+\.o)(?:\s|$)", output))
71+
72+
prebuilt = compiled(run(make))
73+
expected = {"prod_mimalloc.o", "prod_sqlite3.o", "prod_grammar_go.o",
74+
"prod_grammar_json.o", "prod_ts_runtime.o", "prod_lz4.o",
75+
"prod_lz4hc.o", "prod_zstd.o", "unixcoder_blob.o"}
76+
# TRE is a Windows-only production object. All other objects are mandatory.
77+
if prebuilt not in (expected, expected | {"prod_tre.o"}):
78+
raise RuntimeError(f"incomplete prebuild: {sorted(prebuilt)}")
79+
if compiled(run(make)):
80+
raise RuntimeError("unchanged prebuild is not reused")
81+
82+
cases = [
83+
("nested allocator header", ["vendored/mimalloc/include/mimalloc/internal.h"], {"prod_mimalloc.o"}),
84+
("aliased allocator source", ["vendored/mimalloc/src/prim/unix/prim.c"], {"prod_mimalloc.o"}),
85+
("nested grammar header", ["internal/cbm/vendored/grammars/go/nested/value.h"], {"prod_grammar_go.o"}),
86+
("grammar wrapper", ["internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}),
87+
("runtime wrapper", ["internal/cbm/ts_runtime.c"], {"prod_ts_runtime.o"}),
88+
("runtime source", ["internal/cbm/vendored/ts_runtime/src/lib.c"], {"prod_ts_runtime.o"}),
89+
("shared grammar header", ["internal/cbm/vendored/common/scanner.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}),
90+
("unknown grammar family", ["internal/cbm/vendored/grammars/shared/shared.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}),
91+
("LZ4 header", ["internal/cbm/vendored/lz4/lz4.h"], {"prod_lz4.o", "prod_lz4hc.o"}),
92+
("SQLite header", ["vendored/sqlite3/sqlite3.h"], {"prod_sqlite3.o"}),
93+
("zstd header", ["internal/cbm/vendored/zstd/zstd.h"], {"prod_zstd.o"}),
94+
("first-party only", ["src/main.c"], set()),
95+
("duplicate changes", ["internal/cbm/grammar_go.c", "internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}),
96+
]
97+
for label, paths, expected_objects in cases:
98+
write("changed.txt", "\n".join(paths) + "\n")
99+
run([bash, "scripts/ci/codeql-retrace-vendored.sh", "--changed", "changed.txt", "retrace.txt"])
100+
retrace = (work / "retrace.txt").read_text(encoding="utf-8").splitlines()
101+
actual = compiled(run(make + [arg for path in retrace for arg in ("-W", path)]))
102+
if actual != expected_objects:
103+
failures.append(f"{label}: expected {sorted(expected_objects)}, compiled {sorted(actual)}")
104+
else:
105+
print(f"PASS {label}: {len(actual)}/{len(prebuilt)} third-party objects retraced")
106+
if failures:
107+
print("CODEQL RETRACE CONTRACT VIOLATED:\n " + "\n ".join(failures))
108+
sys.exit(1)
109+
print("CodeQL retrace contract OK (real compiler + production Make rules)")
110+
PY

‎tests/test_select_lanes.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -263,15 +263,18 @@ env = {k: v for k, v in os.environ.items() if k != "GITHUB_EVENT_NAME"}
263263
changed = work / "codeql-changed.txt"
264264
changed.write_text("src/main.c\ninternal/cbm/vendored/grammars/go/parser.c\n"
265265
"vendored/sqlite3/sqlite3.c\nvendored/no-such-file.c\n"
266+
"internal/cbm/grammar_json.c\ninternal/cbm/ts_runtime.c\n"
266267
"graph-ui/src/App.tsx\n", encoding="utf-8")
267268
listed = work / "codeql-retrace.txt"
268269
proc = subprocess.run([bash, str(retrace), "--changed", str(changed), str(listed)],
269270
capture_output=True, text=True, env=env)
270271
got = listed.read_text(encoding="utf-8").split() if listed.exists() else None
271-
if proc.returncode != 0 or got != ["internal/cbm/vendored/grammars/go/parser.c",
272+
if proc.returncode != 0 or got != ["internal/cbm/grammar_go.c",
273+
"internal/cbm/grammar_json.c",
274+
"internal/cbm/ts_runtime.c",
272275
"vendored/sqlite3/sqlite3.c"]:
273-
failures.append(f"codeql (c): the retrace helper must list exactly the changed vendored "
274-
f"C files that exist, got {got} (exit {proc.returncode})")
276+
failures.append(f"codeql (c): the retrace helper must list the translation units for "
277+
f"changed vendored code and wrappers, got {got} (exit {proc.returncode})")
275278
for label, args in (("an unreadable change list", ["--changed", str(work / "missing.txt")]),
276279
("no pull request", [])):
277280
listed.write_text("stale\n", encoding="utf-8")

0 commit comments

Comments
 (0)