Skip to content

Commit d8a1002

Browse files
authored
Merge branch 'main' into fix/index-worker-explicit-memory-budget
2 parents bf46b91 + 49d928b commit d8a1002

4 files changed

Lines changed: 285 additions & 8 deletions

File tree

‎.github/workflows/release.yml‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -316,10 +316,23 @@ jobs:
316316
# install.sh and install.ps1 are the highest-consequence non-executable
317317
# bytes we publish: users pipe them straight into a shell.
318318
#
319-
# The selected executables are re-submitted too. That is deliberate and
320-
# nearly free: VirusTotal is content-addressed, so identical bytes return
321-
# the analysis it already holds instead of re-running 70+ engines — the
322-
# same property that made the analysis-id equality check untenable.
319+
# The selected executables are NOT re-submitted. They were scanned as
320+
# candidates, and verify-release-selection.py has just proven every
321+
# published container carries exactly those bytes — identity is settled by
322+
# hash, so a second scan adds no assurance.
323+
#
324+
# It also is not free, contrary to what this comment used to claim.
325+
# Measured on v0.10.5: all eight re-submissions produced a NEW analysis
326+
# (same file-id, timestamp 47 minutes later), and Microsoft's ML engine
327+
# answered differently for two of them within that hour — in opposite
328+
# directions. The release notes then cited one scan while the links showed
329+
# the other. See scripts/ci/exclude-rescanned-selected-objects.sh.
330+
- name: Withhold already-scanned selected executables
331+
run: |
332+
bash scripts/ci/exclude-rescanned-selected-objects.sh \
333+
binaries/objects "$RUNNER_TEMP/release-selection.tsv" \
334+
binaries/virustotal-withheld.tsv
335+
323336
- name: VirusTotal scan of every extracted release object
324337
uses: crazy-max/ghaction-virustotal@936d8c5c00afe97d3d9a1af26d017cfdf26800a2 # v5.0.0
325338
id: virustotal

‎internal/cbm/cbm.c‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -787,17 +787,53 @@ static void cbm_error_regions_push(cbm_error_regions_t *acc, TSNode n) {
787787
acc->count++;
788788
}
789789

790-
static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc) {
790+
/* #1610: a file that does not end with a newline leaves the grammar's
791+
* mandatory line terminator MISSING. That node is ZERO-WIDTH and sits at EOF.
792+
*
793+
* It is not a miss. The parser consumed no source for it — start_byte ==
794+
* end_byte — so by construction nothing was dropped: no construct can live in
795+
* a zero-byte span, and every real instruction above it parsed normally. This
796+
* is a property of the grammar's terminator rule, not of the file.
797+
*
798+
* Flagging it made the verdict arbitrary. Grammars whose terminator token is
799+
* VISIBLE (dockerfile, tcl, fish, gomod, hyprlang) reported parse_partial for
800+
* a missing final newline; grammars whose terminator is HIDDEN (ini, fsharp,
801+
* beancount, requirements, gitignore, sshconfig, kconfig) reported nothing for
802+
* exactly the same omission, because a hidden node is invisible to
803+
* ts_node_child(). Whether a user was told their file was partially parsed
804+
* depended on a grammar-authoring accident.
805+
*
806+
* The cost was not cosmetic: a phantom parse_partial writes a
807+
* "<project>::missed" shadow row, and until #1609 that row made the project
808+
* fail cross-repo validation as both source and target.
809+
*
810+
* Deliberately narrow — ZERO-WIDTH AT EOF ONLY. A MISSING or ERROR node with
811+
* WIDTH still counts even at EOF (a Makefile whose last recipe line is
812+
* unterminated really does lose the recipe), and anything before EOF is
813+
* untouched. */
814+
static bool cbm_is_eof_terminator_miss(TSNode n, int source_len) {
815+
if (!ts_node_is_missing(n) || source_len < 0) {
816+
return false;
817+
}
818+
uint32_t start = ts_node_start_byte(n);
819+
uint32_t end = ts_node_end_byte(n);
820+
return start == end && end == (uint32_t)source_len;
821+
}
822+
823+
static void cbm_collect_error_regions(TSNode n, cbm_error_regions_t *acc, int source_len) {
791824
if (acc->count >= CBM_MAX_ERROR_REGIONS) {
792825
return;
793826
}
794827
uint32_t k = ts_node_child_count(n);
795828
for (uint32_t i = 0; i < k && acc->count < CBM_MAX_ERROR_REGIONS; i++) {
796829
TSNode c = ts_node_child(n, i);
797830
if (ts_node_is_missing(c) || strcmp(ts_node_type(c), "ERROR") == 0) {
831+
if (cbm_is_eof_terminator_miss(c, source_len)) {
832+
continue; /* absent final newline only — nothing was dropped */
833+
}
798834
cbm_error_regions_push(acc, c); /* top-most region; do not descend */
799835
} else if (ts_node_has_error(c)) {
800-
cbm_collect_error_regions(c, acc);
836+
cbm_collect_error_regions(c, acc, source_len);
801837
}
802838
}
803839
}
@@ -1413,7 +1449,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
14131449
* already extract. */
14141450
if (ts_node_has_error(root)) {
14151451
cbm_error_regions_t raw_regs = {{0}, {0}, 0};
1416-
cbm_collect_error_regions(root, &raw_regs);
1452+
cbm_collect_error_regions(root, &raw_regs, source_len);
14171453
if (raw_regs.count > 0) {
14181454
int defs_before = result->defs.count;
14191455
cbm_extract_definitions(&pp_ctx);
@@ -1571,7 +1607,7 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
15711607
if (strcmp(ts_node_type(root), "ERROR") == 0) {
15721608
cbm_error_regions_push(&regs, root); /* whole file unparseable */
15731609
} else {
1574-
cbm_collect_error_regions(root, &regs);
1610+
cbm_collect_error_regions(root, &regs, source_len);
15751611
}
15761612
cbm_subtract_recovered_regions(&regs, &result->defs);
15771613
/* #1071: don't flag a benign function-like-macro call (defined in-file)
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/usr/bin/env bash
2+
# Withhold the selected executables from the full-surface VirusTotal pass.
3+
#
4+
# WHY (measured, not assumed). The verify pass used to submit every extracted
5+
# object, selected executables included, on the stated grounds that "VirusTotal
6+
# is content-addressed, so identical bytes return the analysis it already holds
7+
# instead of re-running 70+ engines". That is not what happens. On the v0.10.5
8+
# release all EIGHT re-submissions produced a NEW analysis: same VirusTotal
9+
# file-id, a timestamp 47 minutes later.
10+
#
11+
# candidate: file-id=2c00f485... ts=1786795957 (12:12:37Z)
12+
# verify : file-id=2c00f485... ts=1786798758 (12:59:18Z)
13+
#
14+
# Re-analysing identical bytes re-rolls a probabilistic classifier, and it
15+
# answered differently in the same hour, in both directions:
16+
#
17+
# 82750cd1 (linux-amd64) microsoft-ml -> clean
18+
# 6d3c5be6 (darwin-arm64) clean -> microsoft-ml
19+
#
20+
# The published release notes then cited the candidate verdict while the linked
21+
# page showed the verify verdict, so the table said "clean" for a binary
22+
# VirusTotal was flagging and vice versa. Nothing about the binaries differed:
23+
# same sha256 in both scans.
24+
#
25+
# The second scan also proved nothing the first did not. Identity is already
26+
# established by hash: verify-release-selection.py reconciles every published
27+
# container to the selected bytes before this step runs, and checksums.txt binds
28+
# the same digests publicly. A re-scan adds no assurance and one more roll.
29+
#
30+
# What still gets scanned is everything the candidate pass never saw: install.sh,
31+
# install.ps1, LICENSE, THIRD_PARTY_NOTICES.md, the MCPB manifest.json and the
32+
# unpacked UI assets. install.sh and install.ps1 are the highest-consequence
33+
# non-executable bytes we publish - users pipe them straight into a shell - and
34+
# that coverage is untouched.
35+
#
36+
# Fails closed: if nothing matches, the selection binding is broken; if nothing
37+
# is left, the surface scan would silently become a no-op.
38+
#
39+
# Usage: exclude-rescanned-selected-objects.sh <objects-dir> <release-selection.tsv> [manifest-out]
40+
set -euo pipefail
41+
42+
OBJECTS_DIR="${1:?usage: exclude-rescanned-selected-objects.sh <objects-dir> <selection.tsv> [manifest-out]}"
43+
SELECTION="${2:?usage: exclude-rescanned-selected-objects.sh <objects-dir> <selection.tsv> [manifest-out]}"
44+
MANIFEST="${3:-}"
45+
46+
test -d "$OBJECTS_DIR" || { echo "error: no objects directory: $OBJECTS_DIR" >&2; exit 1; }
47+
test -s "$SELECTION" || { echo "error: no selection evidence: $SELECTION" >&2; exit 1; }
48+
49+
head -n 1 "$SELECTION" | grep -qx '# cbm-release-selection-v1' || {
50+
echo "error: wrong evidence marker in $SELECTION" >&2; exit 1; }
51+
52+
# selected_sha256 is the 4th column; skip the '#' metadata block and the header.
53+
selected="$(mktemp)"
54+
withheld="$(mktemp)"
55+
trap 'rm -f "$selected" "$withheld"' EXIT
56+
# `|| true` on the grep: with `set -o pipefail` a zero-match grep would abort the
57+
# script here, before the explicit check below could say why. A guard that exits
58+
# silently is not a guard.
59+
awk -F '\t' '/^#/ {next} $1=="target" {next} {print $4}' "$SELECTION" \
60+
| { grep -E '^[0-9a-f]{64}$' || true; } | sort -u > "$selected"
61+
if [ ! -s "$selected" ]; then
62+
echo "error: no selected sha256 values in $SELECTION — the selection" >&2
63+
echo " evidence names no shipped bytes, so nothing can be matched." >&2
64+
exit 1
65+
fi
66+
67+
kept=0
68+
for f in "$OBJECTS_DIR"/*; do
69+
[ -f "$f" ] || continue
70+
sha="$(sha256sum "$f" | awk '{print $1}')"
71+
if grep -qx "$sha" "$selected"; then
72+
printf '%s\t%s\n' "$sha" "$(basename "$f")" >> "$withheld"
73+
rm -f -- "$f"
74+
else
75+
kept=$((kept + 1))
76+
fi
77+
done
78+
79+
count="$(wc -l < "$withheld" | tr -d ' ')"
80+
if [ "$count" -eq 0 ]; then
81+
echo "error: no extracted object matched a selected sha256 — the release" >&2
82+
echo " containers do not carry the bytes the selection recorded." >&2
83+
exit 1
84+
fi
85+
if [ "$kept" -eq 0 ]; then
86+
echo "error: every extracted object was withheld; the full-surface scan" >&2
87+
echo " would cover nothing." >&2
88+
exit 1
89+
fi
90+
91+
if [ -n "$MANIFEST" ]; then
92+
{
93+
echo "# cbm-virustotal-withheld-v1"
94+
echo "# reason=already-scanned-as-candidate"
95+
echo "# evidence=virustotal-candidate-results.tsv"
96+
printf 'sha256\tobject\n'
97+
cat "$withheld"
98+
} > "$MANIFEST"
99+
fi
100+
101+
echo "withheld $count already-scanned selected executable(s); $kept object(s) remain for the surface scan"

‎tests/test_parse_coverage.c‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,128 @@ TEST(c_trailing_recovered_defs_keep_flag) {
247247

248248
/* ── Suite ────────────────────────────────────────────────────────────────── */
249249

250+
/* ── #1610: a missing FINAL NEWLINE is not a parse failure ────────────────────
251+
*
252+
* A file that does not end with "\n" leaves the grammar's mandatory line
253+
* terminator MISSING. That node is ZERO-WIDTH and sits at EOF: the parser
254+
* consumed no source for it, so by construction nothing was dropped — no
255+
* construct can live in a zero-byte span. Every instruction still parses.
256+
*
257+
* Reported on #1610 for Dockerfile, where a reporter proved with a byte-exact
258+
* matrix that the trigger is independent of BOM, CRLF/LF, exec-form vs
259+
* shell-form and file length — it is purely the absent final newline.
260+
*
261+
* It was never Dockerfile-specific: tcl, fish, gomod and hyprlang flag the same
262+
* way, while ini, fsharp, beancount and others do NOT — only because those
263+
* grammars declare the terminator token hidden rather than visible. Whether a
264+
* user saw a phantom parse_partial came down to a grammar-authoring accident.
265+
*
266+
* The cost was not cosmetic: a phantom flag writes a "<project>::missed" shadow
267+
* row, and until #1609 that row removed the whole project from cross-repo
268+
* linking, as source AND as target. */
269+
TEST(dockerfile_missing_final_newline_not_flagged_issue1610) {
270+
const char *src = "FROM mcr.microsoft.com/dotnet/aspnet:8.0\n"
271+
"ENTRYPOINT [\"dotnet\", \"App.dll\"]"; /* deliberately no \n */
272+
CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile");
273+
ASSERT_NOT_NULL(r);
274+
bool flagged = r->parse_incomplete;
275+
cbm_free_result(r);
276+
if (flagged) {
277+
FAIL("a Dockerfile lacking only its final newline must not be parse_partial");
278+
}
279+
PASS();
280+
}
281+
282+
/* The same bytes WITH the newline must stay clean — pins the equivalence the
283+
* reporter's matrix proved, so a future change cannot "fix" one by breaking the
284+
* other. */
285+
TEST(dockerfile_with_final_newline_still_clean_issue1610) {
286+
const char *src = "FROM mcr.microsoft.com/dotnet/aspnet:8.0\n"
287+
"ENTRYPOINT [\"dotnet\", \"App.dll\"]\n";
288+
CBMFileResult *r = do_extract(src, CBM_LANG_DOCKERFILE, "Dockerfile");
289+
ASSERT_NOT_NULL(r);
290+
bool flagged = r->parse_incomplete;
291+
cbm_free_result(r);
292+
if (flagged) {
293+
FAIL("a terminated Dockerfile must not be parse_partial");
294+
}
295+
PASS();
296+
}
297+
298+
/* Language-general, not a Dockerfile patch: these four were each proven to flag
299+
* on a stripped trailing newline. */
300+
TEST(missing_final_newline_not_flagged_across_grammars_issue1610) {
301+
struct {
302+
const char *src;
303+
CBMLanguage lang;
304+
const char *path;
305+
} cases[] = {
306+
{"proc foo {} {}\nproc bar {} {}", CBM_LANG_TCL, "a.tcl"},
307+
{"function foo\n echo hi\nend", CBM_LANG_FISH, "a.fish"},
308+
{"module example.com/m\n\ngo 1.21", CBM_LANG_GOMOD, "go.mod"},
309+
{"general {\n gaps_in = 5\n}", CBM_LANG_HYPRLANG, "hypr.conf"},
310+
};
311+
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
312+
CBMFileResult *r = do_extract(cases[i].src, cases[i].lang, cases[i].path);
313+
ASSERT_NOT_NULL(r);
314+
bool flagged = r->parse_incomplete;
315+
if (flagged) {
316+
fprintf(stderr, " %s flagged: ranges=%s\n", cases[i].path,
317+
r->error_ranges ? r->error_ranges : "(none)");
318+
}
319+
cbm_free_result(r);
320+
if (flagged) {
321+
FAIL("an unterminated final line must not be parse_partial in any grammar");
322+
}
323+
}
324+
PASS();
325+
}
326+
327+
/* GUARD (the reason this suppression is safe rather than convenient): the rule
328+
* is ZERO-WIDTH AT EOF only. A real failure earlier in the file must still be
329+
* reported, and its range must name the broken line — not be swallowed along
330+
* with the terminator. */
331+
TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610) {
332+
/* Built from C_IFDEF_SPLIT, the fixture this suite already proves is
333+
* flagged, with its trailing newline removed. Two conditions now hold at
334+
* once: a genuine width-bearing ERROR mid-file, AND an unterminated last
335+
* line. Suppressing the EOF terminator must not swallow the real one. */
336+
size_t n = strlen(C_IFDEF_SPLIT);
337+
char *unterminated = (char *)malloc(n + 1);
338+
ASSERT_NOT_NULL(unterminated);
339+
memcpy(unterminated, C_IFDEF_SPLIT, n);
340+
unterminated[n - 1] = '\0'; /* drop the final newline */
341+
342+
CBMFileResult *r = do_extract(unterminated, CBM_LANG_C, "split.c");
343+
free(unterminated);
344+
ASSERT_NOT_NULL(r);
345+
bool flagged = r->parse_incomplete;
346+
bool has_ranges = r->error_ranges != NULL;
347+
cbm_free_result(r);
348+
if (!flagged) {
349+
FAIL("a real mid-file parse failure must still be reported when the file also lacks its final newline");
350+
}
351+
if (!has_ranges) {
352+
FAIL("a reported failure must still name its line range");
353+
}
354+
PASS();
355+
}
356+
357+
/* GUARD: a MISSING/ERROR node WITH WIDTH at EOF is a genuine loss and must
358+
* still be flagged. A Makefile whose final recipe line lacks its newline really
359+
* does drop the recipe from the tree — cbm's flag is honest there. */
360+
TEST(width_bearing_error_at_eof_still_flagged_issue1610) {
361+
const char *src = "all:\n\techo hi"; /* no trailing newline; recipe is lost */
362+
CBMFileResult *r = do_extract(src, CBM_LANG_MAKEFILE, "Makefile");
363+
ASSERT_NOT_NULL(r);
364+
bool flagged = r->parse_incomplete;
365+
cbm_free_result(r);
366+
if (!flagged) {
367+
FAIL("a width-bearing parse failure at EOF must still be reported");
368+
}
369+
PASS();
370+
}
371+
250372
SUITE(parse_coverage) {
251373
RUN_TEST(c_ifdef_split_brace_sets_parse_incomplete);
252374
RUN_TEST(c_ifdef_split_brace_neighbors_still_extracted);
@@ -257,4 +379,9 @@ SUITE(parse_coverage) {
257379
RUN_TEST(py_clean_file_not_flagged);
258380
RUN_TEST(error_region_cap_is_honored);
259381
RUN_TEST(c_trailing_recovered_defs_keep_flag);
382+
RUN_TEST(dockerfile_missing_final_newline_not_flagged_issue1610);
383+
RUN_TEST(dockerfile_with_final_newline_still_clean_issue1610);
384+
RUN_TEST(missing_final_newline_not_flagged_across_grammars_issue1610);
385+
RUN_TEST(real_error_before_eof_still_flagged_without_final_newline_issue1610);
386+
RUN_TEST(width_bearing_error_at_eof_still_flagged_issue1610);
260387
}

0 commit comments

Comments
 (0)