Skip to content

Commit 35ddda9

Browse files
authored
Merge pull request DeusData#1774 from DeusData/distill/nomic-blob-generator
fix(build): stop blob regeneration from reverting the ELF stack hardening
2 parents 7b6363d + 978ac7d commit 35ddda9

3 files changed

Lines changed: 121 additions & 3 deletions

File tree

‎scripts/extract_nomic_vectors.py‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -332,9 +332,18 @@ def write_vectors_h(path: str, token_count: int, dim: int, incbin_path: str):
332332

333333

334334
def write_blob_s(path: str, incbin_path: str):
335-
"""Write assembler .incbin directive."""
336-
with open(path, "w") as f:
337-
f.write(f"""/* nomic-embed-code vector blob embedded via assembler. */
335+
"""Write the assembler .incbin wrapper for every target object format.
336+
337+
Must stay byte-identical to the tracked vendored/nomic/code_vectors_blob.S:
338+
tests/test_nomic_blob_generator_contract.sh fails if the two drift. The ELF
339+
branch's .note.GNU-stack is load-bearing -- regenerating without it puts an
340+
executable stack back into every Linux release binary.
341+
"""
342+
with open(path, "w", encoding="utf-8") as f:
343+
f.write(f"""/* nomic-embed-code vector blob embedded via assembler.
344+
* Cross-platform: macOS (Mach-O) vs Linux (ELF) vs Windows (COFF). */
345+
346+
#if defined(__APPLE__)
338347
.section __DATA,__const
339348
.globl _PRETRAINED_VECTOR_BLOB
340349
.globl _PRETRAINED_VECTOR_BLOB_LEN
@@ -347,6 +356,46 @@ def write_blob_s(path: str, incbin_path: str):
347356
.p2align 2
348357
_PRETRAINED_VECTOR_BLOB_LEN:
349358
.long _PRETRAINED_VECTOR_BLOB_END - _PRETRAINED_VECTOR_BLOB
359+
360+
#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__)
361+
.section .rdata,"dr"
362+
.globl PRETRAINED_VECTOR_BLOB
363+
.globl PRETRAINED_VECTOR_BLOB_LEN
364+
.p2align 4
365+
PRETRAINED_VECTOR_BLOB:
366+
.incbin "{incbin_path}"
367+
PRETRAINED_VECTOR_BLOB_END:
368+
369+
.section .rdata,"dr"
370+
.p2align 2
371+
PRETRAINED_VECTOR_BLOB_LEN:
372+
.long PRETRAINED_VECTOR_BLOB_END - PRETRAINED_VECTOR_BLOB
373+
374+
#else
375+
/* WHY: an ELF object that carries no .note.GNU-stack tells the linker
376+
* nothing about its stack requirement, and GNU ld then assumes the WORST
377+
* for the whole link — every Linux release binary shipped GNU_STACK RWE
378+
* because of this one omission. This is the only assembly source in the
379+
* build, so it alone decided that property. The note must stay even though
380+
* a blob of constant data obviously never executes: absence is the signal,
381+
* not the contents. -Wl,-z,noexecstack in the link flags enforces the
382+
* outcome, and scripts/ci/check-binary-composition.sh fails the release if
383+
* an executable stack ever comes back. */
384+
.section .note.GNU-stack,"",@progbits
385+
386+
.section .rodata,"a",@progbits
387+
.globl PRETRAINED_VECTOR_BLOB
388+
.globl PRETRAINED_VECTOR_BLOB_LEN
389+
.p2align 4
390+
PRETRAINED_VECTOR_BLOB:
391+
.incbin "{incbin_path}"
392+
PRETRAINED_VECTOR_BLOB_END:
393+
394+
.section .rodata,"a",@progbits
395+
.p2align 2
396+
PRETRAINED_VECTOR_BLOB_LEN:
397+
.long PRETRAINED_VECTOR_BLOB_END - PRETRAINED_VECTOR_BLOB
398+
#endif
350399
""")
351400
print(f" {path}: written")
352401

‎scripts/test.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,9 @@ bash "$ROOT/tests/test_runtime_isolation_contract.sh"
263263
echo "=== Step 0u: shell line-ending contract ==="
264264
bash "$ROOT/tests/test_shell_line_endings.sh"
265265

266+
echo "=== Step 0v: nomic blob generator contract ==="
267+
bash "$ROOT/tests/test_nomic_blob_generator_contract.sh"
268+
266269
# Verify compiler supports target arch
267270
verify_compiler "$CC"
268271

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/usr/bin/env bash
2+
# Contract: scripts/extract_nomic_vectors.py must still emit the tracked
3+
# vendored/nomic/code_vectors_blob.S byte for byte.
4+
#
5+
# WHY this exists: the .S is a GENERATED file that was hand-edited to add the
6+
# ELF .note.GNU-stack section. Without that note GNU ld assumes the whole link
7+
# needs an executable stack, and every Linux release binary shipped GNU_STACK
8+
# RWE because of the omission. The generator kept emitting only the Mach-O
9+
# branch, so the next regeneration would have silently reverted the hardening
10+
# and reintroduced the defect. check-binary-composition.sh catches that at
11+
# release time; this catches it at edit time.
12+
set -euo pipefail
13+
14+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
15+
GEN="$ROOT/scripts/extract_nomic_vectors.py"
16+
TRACKED="$ROOT/vendored/nomic/code_vectors_blob.S"
17+
18+
for f in "$GEN" "$TRACKED"; do
19+
if [ ! -f "$f" ]; then
20+
echo "FAIL: missing $f" >&2
21+
exit 1
22+
fi
23+
done
24+
25+
# The generator holds the .S as an f-string whose only placeholder is the
26+
# incbin path, so the tracked file with that path re-parameterized must appear
27+
# in it verbatim. Comparing the template rather than running the generator
28+
# keeps this test free of the torch/transformers import the script needs.
29+
python3 - "$GEN" "$TRACKED" <<'PYEOF'
30+
import sys
31+
32+
gen_path, tracked_path = sys.argv[1], sys.argv[2]
33+
34+
with open(gen_path, encoding="utf-8") as fh:
35+
generator = fh.read()
36+
with open(tracked_path, encoding="utf-8") as fh:
37+
tracked = fh.read()
38+
39+
INCBIN = "vendored/nomic/code_vectors.bin"
40+
if INCBIN not in tracked:
41+
sys.exit("FAIL: %s no longer references %s" % (tracked_path, INCBIN))
42+
if "{" in tracked or "}" in tracked:
43+
sys.exit("FAIL: %s gained a brace; the generator f-string would need "
44+
"escaping and this contract can no longer compare them directly"
45+
% tracked_path)
46+
47+
template = tracked.replace(INCBIN, "{incbin_path}")
48+
49+
if template not in generator:
50+
sys.exit(
51+
"FAIL: scripts/extract_nomic_vectors.py no longer emits the tracked\n"
52+
" vendored/nomic/code_vectors_blob.S.\n"
53+
" Regenerating the blob wrapper would overwrite the tracked file\n"
54+
" with different content. Update write_blob_s() so its f-string\n"
55+
" matches the tracked .S with the incbin path parameterized.")
56+
57+
# The note is the whole point; assert it explicitly so a future edit that keeps
58+
# the files in sync while dropping the note still fails loudly.
59+
NOTE = '.section .note.GNU-stack,"",@progbits'
60+
for label, blob in (("tracked .S", tracked), ("generator template", generator)):
61+
if NOTE not in blob:
62+
sys.exit("FAIL: %s lost the ELF %s section -- Linux binaries would ship "
63+
"an executable stack again" % (label, NOTE))
64+
65+
print("ok: generator template is byte-identical to the tracked blob wrapper")
66+
PYEOF

0 commit comments

Comments
 (0)