Skip to content

Commit e24ce80

Browse files
committed
Merge origin/main into fix/search-code-cancellation
Trivial insertion-point collision in tests/test_mcp.c: both this branch and main (DeusData#1704's UTF-8 pin) add a Windows search_code test at the same spot. Both tests kept.
2 parents 506151f + 9de6831 commit e24ce80

45 files changed

Lines changed: 2730 additions & 291 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitattributes‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# Shell scripts must stay LF: with git's Windows default (core.autocrlf=true)
2+
# they check out as CRLF and bash inside the build containers rejects them.
3+
*.sh text eol=lf
4+
15
# Vendored tree-sitter grammars — machine-generated, hide from stats and diffs
26
internal/cbm/vendored/grammars/**/parser.c linguist-generated=true diff=false
37
internal/cbm/vendored/grammars/**/scanner.c linguist-generated=true diff=false

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@ jobs:
350350
VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv
351351
VT_ASSOCIATIONS: binaries/associations.tsv
352352
VT_RESULTS_PATH: binaries/vt-results.tsv
353+
VT_WITHHELD: binaries/virustotal-withheld.tsv
353354
MIN_ENGINES: 50
354355
run: scripts/ci/check-virustotal.sh
355356

@@ -362,6 +363,7 @@ jobs:
362363
binaries/associations.tsv
363364
binaries/scan-set.tsv
364365
binaries/vt-results.tsv
366+
binaries/virustotal-withheld.tsv
365367
if-no-files-found: warn
366368

367369
- name: Publish durable public VirusTotal evidence

‎Makefile.cbm‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -729,7 +729,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
729729

730730
# ── Targets ──────────────────────────────────────────────────────
731731

732-
.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security
732+
.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security
733733

734734
$(BUILD_DIR):
735735
mkdir -p $(BUILD_DIR)
@@ -1103,6 +1103,12 @@ lint-tidy:
11031103
@echo "=== clang-tidy ==="
11041104
@$(CLANG_TIDY) --quiet $(LINT_SRCS) -- $(CFLAGS_COMMON) $(SYSROOT_FLAG)
11051105

1106+
# clang-tidy, scoped to this commit's staged changes only (scripts/lint-tidy-diff.sh).
1107+
# The pre-commit hook runs this instead of the full lint-tidy sweep above, so a
1108+
# clean commit is not blocked by pre-existing findings elsewhere in the tree.
1109+
lint-tidy-diff:
1110+
@scripts/lint-tidy-diff.sh "$(CLANG_TIDY)" $(LINT_SRCS) -- $(CFLAGS_COMMON) $(SYSROOT_FLAG)
1111+
11061112
# cppcheck: complementary analysis (config in .cppcheck)
11071113
lint-cppcheck:
11081114
@echo "=== cppcheck ==="

‎internal/cbm/extract_defs.c‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3747,6 +3747,14 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
37473747
def.is_test = true;
37483748
}
37493749

3750+
// A function/method defined in a file cbm treats as a test file is itself
3751+
// a test, regardless of its own name (tests/helpers/fixtures.c has none
3752+
// of the test_/_test naming conventions, but every function in it is
3753+
// still test code) (#1294). OR'd so the Rust attribute check above, which
3754+
// additionally catches #[test] functions embedded in an otherwise regular
3755+
// .rs file, is never downgraded by this.
3756+
def.is_test = def.is_test || ctx->result->is_test_file;
3757+
37503758
// Docstring
37513759
def.docstring = extract_docstring(a, node, ctx->source, ctx->language);
37523760

@@ -4728,6 +4736,10 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, TSNode class_node,
47284736
// MinHash fingerprint
47294737
compute_fingerprint(ctx, &def, child);
47304738

4739+
// A method on a class defined in a test file (e.g. a JUnit/pytest
4740+
// TestFoo.test_bar) is itself a test, same as free functions (#1294).
4741+
def.is_test = def.is_test || ctx->result->is_test_file;
4742+
47314743
cbm_defs_push(&ctx->result->defs, a, def);
47324744
}
47334745

‎internal/cbm/helpers.c‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,19 @@ bool cbm_is_test_file(const char *rel_path, CBMLanguage lang) {
295295
}
296296
const char *base = path_basename(rel_path);
297297

298+
/* Directory-based, language-agnostic: a file under a conventional test
299+
* directory is a test file regardless of its own basename (e.g.
300+
* tests/helpers/fixtures.c, which no per-language suffix/prefix rule
301+
* below would otherwise catch). Mirrors the directory set cbm_is_test_path
302+
* (src/pipeline/pass_tests.c) already uses for TESTS-edge detection, so
303+
* the extraction-time is_test_file flag agrees with it (#1294). */
304+
if (strstr(rel_path, "__tests__/") || strstr(rel_path, "/tests/") ||
305+
strstr(rel_path, "/test/") || strstr(rel_path, "/spec/") ||
306+
has_prefix(rel_path, "tests/") || has_prefix(rel_path, "test/") ||
307+
has_prefix(rel_path, "spec/") || has_prefix(rel_path, "__tests__/")) {
308+
return true;
309+
}
310+
298311
switch (lang) {
299312
case CBM_LANG_GO:
300313
return has_suffix(base, "_test.go");

‎scripts/ci/check-virustotal.sh‎

Lines changed: 88 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ class ExpectedObject:
106106
size: int
107107
association_count: int
108108
association_kinds: frozenset[str]
109+
withheld: bool
109110
110111
@property
111112
def requires_microsoft(self) -> bool:
@@ -188,7 +189,54 @@ def parse_versioned_tsv(
188189
return metadata, rows
189190
190191
191-
def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]:
192+
def load_withheld(raw_path: str) -> Dict[str, str]:
193+
"""Objects deliberately removed from the surface scan (VT_WITHHELD).
194+
195+
exclude-rescanned-selected-objects.sh deletes the SELECTED executables from
196+
the objects directory before the upload — their bytes were already scanned
197+
as release candidates, identity is settled by sha256, and re-submitting
198+
identical bytes re-rolls a probabilistic classifier (measured on v0.10.5:
199+
verdicts flipped in both directions within the hour). The gate accepts a
200+
withheld object only when this manifest vouches for it BY HASH; everything
201+
else keeps the strict on-disk contract. An absent manifest keeps the
202+
original behavior everywhere else this gate runs (candidate stage,
203+
dry-run), where nothing is ever withheld."""
204+
path = pathlib.Path(raw_path).absolute()
205+
if path.is_symlink() or not path.is_file():
206+
raise GateError(f"withheld manifest is not a regular file: {path}")
207+
if path.stat().st_size > 1024 * 1024:
208+
raise GateError(f"withheld manifest is unexpectedly large: {path}")
209+
lines = path.read_text(encoding="utf-8").splitlines()
210+
if not lines or lines[0] != "# cbm-virustotal-withheld-v1":
211+
raise GateError("withheld manifest marker is missing")
212+
cursor = 1
213+
metadata: Dict[str, str] = {}
214+
while cursor < len(lines) and lines[cursor].startswith("# "):
215+
key, separator, value = lines[cursor][2:].partition("=")
216+
if not separator or not key or key in metadata or not value:
217+
raise GateError(f"malformed withheld metadata: {lines[cursor]}")
218+
metadata[key] = value
219+
cursor += 1
220+
if metadata.get("reason") != "already-scanned-as-candidate":
221+
raise GateError("withheld manifest does not state the accepted reason")
222+
if cursor >= len(lines) or lines[cursor] != "sha256\tobject":
223+
raise GateError("withheld manifest header is malformed")
224+
withheld: Dict[str, str] = {}
225+
for line in lines[cursor + 1 :]:
226+
sha256, separator, name = line.partition("\t")
227+
if not separator or SHA256_RE.fullmatch(sha256) is None or not name or "/" in name:
228+
raise GateError(f"malformed withheld row: {line}")
229+
if sha256 in withheld:
230+
raise GateError(f"duplicate withheld sha256: {sha256}")
231+
withheld[sha256] = name
232+
if not withheld:
233+
raise GateError("withheld manifest names no objects")
234+
return withheld
235+
236+
237+
def load_expected(
238+
path: pathlib.Path, withheld: Dict[str, str]
239+
) -> Tuple[List[ExpectedObject], int]:
192240
metadata, rows = parse_versioned_tsv(
193241
path,
194242
marker="cbm-release-scan-set-v2",
@@ -230,15 +278,26 @@ def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]:
230278
):
231279
raise GateError(f"unassociated object in expected set: {scan_path}")
232280
local_path = path.parent.joinpath(*pure.parts)
233-
try:
234-
mode = local_path.lstat().st_mode
235-
except FileNotFoundError as error:
236-
raise GateError(f"expected scan object is missing: {scan_path}") from error
237-
if not stat.S_ISREG(mode):
238-
raise GateError(f"expected scan object is not a regular file: {scan_path}")
239-
actual_size = local_path.stat().st_size
240-
if actual_size != size or sha256_file(local_path) != sha256:
241-
raise GateError(f"expected scan object changed after extraction: {scan_path}")
281+
is_withheld = sha256 in withheld
282+
if is_withheld:
283+
if withheld[sha256] != pure.parts[1]:
284+
raise GateError(
285+
f"withheld manifest names a different object for this hash: {scan_path}"
286+
)
287+
if os.path.lexists(local_path):
288+
raise GateError(
289+
f"withheld object is still present in the scan directory: {scan_path}"
290+
)
291+
else:
292+
try:
293+
mode = local_path.lstat().st_mode
294+
except FileNotFoundError as error:
295+
raise GateError(f"expected scan object is missing: {scan_path}") from error
296+
if not stat.S_ISREG(mode):
297+
raise GateError(f"expected scan object is not a regular file: {scan_path}")
298+
actual_size = local_path.stat().st_size
299+
if actual_size != size or sha256_file(local_path) != sha256:
300+
raise GateError(f"expected scan object changed after extraction: {scan_path}")
242301
association_total += association_count
243302
objects.append(
244303
ExpectedObject(
@@ -248,10 +307,17 @@ def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]:
248307
size=size,
249308
association_count=association_count,
250309
association_kinds=frozenset(kinds),
310+
withheld=is_withheld,
251311
)
252312
)
253313
if association_total != associations:
254314
raise GateError("scan-set association counts do not match manifest metadata")
315+
expected_hashes = {item.sha256 for item in objects}
316+
spurious = sorted(set(withheld) - expected_hashes)
317+
if spurious:
318+
raise GateError(f"withheld manifest names hashes outside the expected scan set: {spurious}")
319+
if all(item.withheld for item in objects):
320+
raise GateError("every expected object is withheld; the surface scan would cover nothing")
255321
return objects, associations
256322
257323
@@ -363,6 +429,7 @@ def parse_action_output(
363429
objects: Sequence[ExpectedObject],
364430
manifest: pathlib.Path,
365431
) -> List[Submission]:
432+
objects = [item for item in objects if not item.withheld]
366433
if not raw:
367434
raise GateError("VirusTotal action output is empty")
368435
aliases = output_aliases(objects, manifest)
@@ -697,7 +764,10 @@ def main() -> None:
697764
raise GateError(f"unsafe pre-existing VT results path: {results_path}")
698765
results_path.unlink()
699766
700-
expected, associations = load_expected(expected_manifest)
767+
withheld_path = os.environ.get("VT_WITHHELD", "")
768+
withheld = load_withheld(withheld_path) if withheld_path else {}
769+
770+
expected, associations = load_expected(expected_manifest, withheld)
701771
validate_associations(
702772
associations_manifest,
703773
objects=expected,
@@ -708,6 +778,13 @@ def main() -> None:
708778
objects=expected,
709779
manifest=expected_manifest,
710780
)
781+
withheld_count = sum(1 for item in expected if item.withheld)
782+
if withheld_count:
783+
print(
784+
f"=== {withheld_count} expected object(s) withheld from the surface scan "
785+
f"(already scanned as candidates; identity settled by sha256, manifest: "
786+
f"{withheld_path}) ==="
787+
)
711788
print(
712789
f"=== VirusTotal exact-set gate: {len(submissions)} distinct objects, "
713790
f"{associations} associations, >= {min_engines} decisive engines each ==="

‎scripts/hooks/pre-commit‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@ set -euo pipefail
77
# git config core.hooksPath scripts/hooks
88

99
echo "pre-commit: running all linters in parallel..."
10-
make -j3 -f Makefile.cbm lint
10+
# clang-tidy runs diff-scoped (lint-tidy-diff, see scripts/lint-tidy-diff.sh):
11+
# the full lint-tidy sweep surfaces thousands of pre-existing findings that
12+
# have nothing to do with this commit and would block every contributor who
13+
# has clang-tidy on PATH. `make lint-tidy` remains available for a full audit.
14+
make -j3 -f Makefile.cbm lint-ci lint-tidy-diff
1115

1216
echo "pre-commit: security audit (source-level)..."
1317
scripts/security-audit.sh

‎scripts/lint-tidy-diff.sh‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env bash
2+
# lint-tidy-diff.sh — clang-tidy scoped to the lines this commit changes.
3+
#
4+
# `make lint-tidy` (the full sweep) checks the entire LINT_SRCS tree, which
5+
# currently surfaces ~5,100 pre-existing findings unrelated to any given
6+
# commit (see Makefile.cbm's `lint-tidy` target). That is fine for a manual
7+
# deep audit but makes the pre-commit hook unusable: it blocks every commit,
8+
# including ones that touch files with zero findings on the touched lines.
9+
#
10+
# This script runs the same clang-tidy binary/config against the SAME staged
11+
# .c files, but passes clang-tidy's own `-line-filter` so only lines the
12+
# commit actually added/modified can produce a diagnostic. Pre-existing
13+
# findings on untouched lines — in the same file or elsewhere — are not
14+
# reported.
15+
#
16+
# Invoked by `make lint-tidy-diff` (scripts/hooks/pre-commit), which supplies
17+
# the resolved clang-tidy binary, then LINT_SRCS, then `--`, then the
18+
# compiler flags — the same argv shape as the `lint-tidy` target itself.
19+
#
20+
# Known limitation: only staged *.c files that are members of LINT_SRCS are
21+
# checked (matching the full sweep's own scope: vendored code and tests/ are
22+
# never clang-tidy'd). clang-tidy reports a header's diagnostics under
23+
# whatever path it was #include-d with, which is not reliably the same
24+
# string as its path in the working tree, so a commit that touches only a
25+
# header is not covered here even if a .c file including it also changed;
26+
# run `make lint-tidy` for a full pass in that case.
27+
28+
set -euo pipefail
29+
30+
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
31+
cd "$ROOT"
32+
33+
if [ "$#" -lt 1 ]; then
34+
echo "usage: lint-tidy-diff.sh <clang-tidy-bin> [LINT_SRCS...] -- [CFLAGS...]" >&2
35+
exit 2
36+
fi
37+
38+
CLANG_TIDY="$1"
39+
shift
40+
41+
LINT_SRCS=()
42+
while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do
43+
LINT_SRCS+=("$1")
44+
shift
45+
done
46+
[ "$#" -gt 0 ] && shift # drop the `--` separator
47+
CFLAGS=("$@")
48+
49+
# bash 3.2 (macOS default, see scripts/check-lsp-originality.sh) has neither
50+
# `mapfile` nor associative arrays — read loop + linear membership check.
51+
is_lint_src() {
52+
local needle="$1" f
53+
[ "${#LINT_SRCS[@]}" -eq 0 ] && return 1
54+
for f in "${LINT_SRCS[@]}"; do
55+
[ "$f" = "$needle" ] && return 0
56+
done
57+
return 1
58+
}
59+
60+
# Staged .c files this commit actually changed (added/copied/modified/
61+
# renamed — a delete has no new lines to check) that are members of
62+
# LINT_SRCS.
63+
CHANGED=()
64+
while IFS= read -r f; do
65+
[ -n "$f" ] && is_lint_src "$f" && CHANGED+=("$f")
66+
done < <(git diff --cached --name-only --diff-filter=ACMR -- '*.c')
67+
68+
if [ "${#CHANGED[@]}" -eq 0 ]; then
69+
echo "lint-tidy-diff: no staged changes to a linted .c file, skipping clang-tidy"
70+
exit 0
71+
fi
72+
73+
# For each changed file, turn its staged hunks into a clang-tidy line-filter
74+
# range: `@@ -a,b +c,d @@` -> lines [c, c+d-1]. Zero-context (-U0) so a range
75+
# never includes untouched, pre-existing lines. Pure-deletion hunks (d == 0)
76+
# add no new lines and are skipped.
77+
LINE_FILTER='['
78+
FILTER_SEP=''
79+
TARGETS=()
80+
for f in "${CHANGED[@]}"; do
81+
[ -f "$f" ] || continue # renamed away / not present in the working tree
82+
ranges=''
83+
range_sep=''
84+
while IFS= read -r hunk; do
85+
if [[ "$hunk" =~ ^@@\ -[0-9]+(,[0-9]+)?\ \+([0-9]+)(,([0-9]+))?\ @@ ]]; then
86+
new_start="${BASH_REMATCH[2]}"
87+
new_count="${BASH_REMATCH[4]:-1}"
88+
[ "$new_count" -eq 0 ] && continue
89+
new_end=$((new_start + new_count - 1))
90+
ranges+="${range_sep}[${new_start},${new_end}]"
91+
range_sep=','
92+
fi
93+
done < <(git diff --cached -U0 --diff-filter=ACMR -- "$f" | grep -E '^@@ ')
94+
[ -z "$ranges" ] && continue
95+
96+
LINE_FILTER+="${FILTER_SEP}{\"name\":\"${f}\",\"lines\":[${ranges}]}"
97+
FILTER_SEP=','
98+
TARGETS+=("$f")
99+
done
100+
LINE_FILTER+=']'
101+
102+
if [ "${#TARGETS[@]}" -eq 0 ]; then
103+
echo "lint-tidy-diff: staged .c changes are deletion-only, skipping clang-tidy"
104+
exit 0
105+
fi
106+
107+
echo "lint-tidy-diff: ${#TARGETS[@]} changed file(s), diff-scoped clang-tidy"
108+
"$CLANG_TIDY" --quiet "-line-filter=${LINE_FILTER}" "${TARGETS[@]}" -- "${CFLAGS[@]}"

‎scripts/security-fuzz-random.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@ fi
2727

2828
echo "=== Fuzz Testing ($DURATION seconds) ==="
2929

30-
FUZZ_TMPDIR=$(mktemp -d)
31-
trap 'rm -rf "$FUZZ_TMPDIR"' EXIT
30+
# shellcheck source=test-runtime.sh
31+
source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
32+
cbm_test_runtime_init
33+
FUZZ_TMPDIR="$CBM_TEST_RUNTIME_ROOT"
34+
trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
3235

3336
CRASHES=0
3437
ITERATIONS=0

0 commit comments

Comments
 (0)